2024-09-21T09:32:43.624 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-21T09:32:43.644 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T09:32:43.730 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913659 branch: main description: upgrade/cephfs/nofs/{bluestore-bitmap centos_9.stream conf/{client mds mgr mon osd} no-mds-cluster overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn pg_health} tasks/{0-from/squid 1-upgrade}} email: ceph-qa@ceph.com first_in_suite: false job_id: '7913659' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'install ceph/squid latest ' name: teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: main ceph: cluster-conf: client: client mount timeout: 600 debug client: 20 debug ms: 1 rados mon op timeout: 900 rados osd op timeout: 900 mds: debug mds: 20 debug mds balancer: 20 debug ms: 2 mds debug frag: true mds debug scatterstat: true mds op complaint time: 180 mds verify scatter: true osd op complaint time: 180 rados mon op timeout: 900 rados osd op timeout: 900 mon: mon op complaint time: 120 mon warn on pool no app: false osd: osd op complaint time: 180 conf: global: bluestore warn on legacy statfs: false bluestore warn on no per pool omap: false mon pg warn min per osd: 0 mgr: client mount timeout: 30 debug client: 20 debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 mon down mkfs grace: 300 mon warn on osd down out interval zero: false osd: bdev async discard: true bdev enable discard: true bluestore allocator: bitmap bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - FS_DEGRADED - fs.*is degraded - filesystem is degraded - FS_INLINE_DATA_DEPRECATED - FS_WITH_FAILED_MDS - MDS_ALL_DOWN - filesystem is offline - MDS_DAMAGE - MDS_DEGRADED - MDS_FAILED - MDS_INSUFFICIENT_STANDBY - insufficient standby MDS daemons available - MDS_UP_LESS_THAN_MAX - online, but wants - filesystem is online with fewer MDS than max_mds - POOL_APP_NOT_ENABLED - do not have an application enabled - overall HEALTH_ - Replacing daemon - deprecated feature inline_data - overall HEALTH_ - \(OSD_DOWN\) - \(OSD_ - but it is still running - is not responding - PG_AVAILABILITY - PG_DEGRADED - Reduced data availability - Degraded data redundancy - scrub mismatch - ScrubResult - wrongly marked - \(POOL_APP_NOT_ENABLED\) - \(SLOW_OPS\) - overall HEALTH_ - \(MON_MSGR2_NOT_ENABLED\) - slow request sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} osd: bdev async discard: true bdev enable discard: true bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs install: ceph: flavor: default sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: main sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 owner: scheduled_teuthology@teuthology priority: 850 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.b - mon.c - mgr.x - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 seed: 5357 sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 sleep_before_teardown: 0 subset: 31/32 suite: upgrade suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 targets: smithi092.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBADqCvZHDIzgj3Qo8lQoI5Nwj8MJWM6J29P8Qgrc7j6VhXOZ8AyH90DS7pd7rxbDEa89Dzqyg9hYQfqgF1mfgEw= tasks: - install: branch: squid exclude_packages: - librados3 - ceph-mgr-dashboard - ceph-mgr-diskprediction-local - ceph-mgr-rook - ceph-mgr-cephadm - cephadm - ceph-volume extra_packages: - librados2 - print: '**** done installing squid' - ceph: conf: global: mon warn on pool no app: false ms bind msgr2: false log-ignorelist: - overall HEALTH_ - \(FS_ - \(MDS_ - \(OSD_ - \(MON_DOWN\) - \(CACHE_POOL_ - \(POOL_ - \(MGR_DOWN\) - \(PG_ - \(SMALLER_PGP_NUM\) - Monitor daemon marked osd - Behind on trimming - Manager daemon - exec: osd.0: - ceph osd set-require-min-compat-client squid - print: '**** done ceph' - print: '*** upgrading, no cephfs present' - exec: mon.a: - ceph fs dump - install.upgrade: mon.a: null - print: '**** done install.upgrade' - ceph.restart: daemons: - mon.* - mgr.* mon-health-to-clog: false wait-for-healthy: false - ceph.healthy: null - ceph.restart: daemons: - osd.* wait-for-healthy: false wait-for-osds-up: true - exec: mon.a: - ceph versions - ceph osd dump -f json-pretty - ceph fs dump - ceph osd require-osd-release squid - for f in `ceph osd pool ls` ; do ceph osd pool set $f pg_autoscale_mode off ; done - ceph.healthy: null - print: '**** done ceph.restart' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-21_03:08:26 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2902 2024-09-21T09:32:43.730 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa; will attempt to use it 2024-09-21T09:32:43.731 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks 2024-09-21T09:32:43.731 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-21T09:32:43.732 INFO:teuthology.task.internal:Checking packages... 2024-09-21T09:32:43.751 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '62da54b4b83a96ca9976b7e1d0c8272a564a2208' 2024-09-21T09:32:43.751 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:32:43.751 INFO:teuthology.packaging:ref: None 2024-09-21T09:32:43.751 INFO:teuthology.packaging:tag: None 2024-09-21T09:32:43.751 INFO:teuthology.packaging:branch: main 2024-09-21T09:32:43.751 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:32:43.752 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=main 2024-09-21T09:32:43.968 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-5092.g62da54b4 2024-09-21T09:32:43.969 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-21T09:32:43.976 INFO:teuthology.task.internal:no buildpackages task found 2024-09-21T09:32:43.976 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-21T09:32:44.012 INFO:teuthology.task.internal:Saving configuration 2024-09-21T09:32:44.028 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-21T09:32:44.035 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-21T09:32:44.060 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi092.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913659', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-21 09:27:53.618938', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBADqCvZHDIzgj3Qo8lQoI5Nwj8MJWM6J29P8Qgrc7j6VhXOZ8AyH90DS7pd7rxbDEa89Dzqyg9hYQfqgF1mfgEw='} 2024-09-21T09:32:44.060 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-21T09:32:44.067 INFO:teuthology.task.internal:roles: ubuntu@smithi092.front.sepia.ceph.com - ['mon.a', 'mon.b', 'mon.c', 'mgr.x', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3'] 2024-09-21T09:32:44.067 INFO:teuthology.run_tasks:Running task console_log... 2024-09-21T09:32:44.112 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fb89abac160>, signals=[15]) 2024-09-21T09:32:44.112 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-21T09:32:44.118 INFO:teuthology.task.internal:Opening connections... 2024-09-21T09:32:44.118 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi092.front.sepia.ceph.com 2024-09-21T09:32:44.119 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi092.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:32:44.194 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-21T09:32:44.200 DEBUG:teuthology.orchestra.run.smithi092:> uname -m 2024-09-21T09:32:44.276 INFO:teuthology.orchestra.run.smithi092.stdout:x86_64 2024-09-21T09:32:44.276 DEBUG:teuthology.orchestra.run.smithi092:> cat /etc/os-release 2024-09-21T09:32:44.334 INFO:teuthology.orchestra.run.smithi092.stdout:NAME="CentOS Stream" 2024-09-21T09:32:44.334 INFO:teuthology.orchestra.run.smithi092.stdout:VERSION="9" 2024-09-21T09:32:44.334 INFO:teuthology.orchestra.run.smithi092.stdout:ID="centos" 2024-09-21T09:32:44.334 INFO:teuthology.orchestra.run.smithi092.stdout:ID_LIKE="rhel fedora" 2024-09-21T09:32:44.334 INFO:teuthology.orchestra.run.smithi092.stdout:VERSION_ID="9" 2024-09-21T09:32:44.334 INFO:teuthology.orchestra.run.smithi092.stdout:PLATFORM_ID="platform:el9" 2024-09-21T09:32:44.334 INFO:teuthology.orchestra.run.smithi092.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-21T09:32:44.335 INFO:teuthology.orchestra.run.smithi092.stdout:ANSI_COLOR="0;31" 2024-09-21T09:32:44.335 INFO:teuthology.orchestra.run.smithi092.stdout:LOGO="fedora-logo-icon" 2024-09-21T09:32:44.335 INFO:teuthology.orchestra.run.smithi092.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-21T09:32:44.335 INFO:teuthology.orchestra.run.smithi092.stdout:HOME_URL="https://centos.org/" 2024-09-21T09:32:44.335 INFO:teuthology.orchestra.run.smithi092.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-21T09:32:44.335 INFO:teuthology.orchestra.run.smithi092.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-21T09:32:44.335 INFO:teuthology.orchestra.run.smithi092.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-21T09:32:44.336 INFO:teuthology.lock.ops:Updating smithi092.front.sepia.ceph.com on lock server 2024-09-21T09:32:44.353 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-21T09:32:44.362 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-21T09:32:44.379 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-21T09:32:44.379 DEBUG:teuthology.orchestra.run.smithi092:> test '!' -e /home/ubuntu/cephtest 2024-09-21T09:32:44.396 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-21T09:32:44.402 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-21T09:32:44.402 DEBUG:teuthology.orchestra.run.smithi092:> test -z $(ls -A /var/lib/ceph) 2024-09-21T09:32:44.540 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-21T09:32:44.570 INFO:teuthology.run_tasks:Running task kernel... 2024-09-21T09:32:44.587 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-21T09:32:44.587 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-21T09:32:44.588 DEBUG:teuthology.orchestra.run.smithi092:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-21T09:32:44.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:32:44.605 DEBUG:teuthology.orchestra.run.smithi092:> uname -r 2024-09-21T09:32:44.662 INFO:teuthology.orchestra.run.smithi092.stdout:5.14.0-503.el9.x86_64 2024-09-21T09:32:44.663 INFO:teuthology.task.kernel:Running kernel on smithi092: 5.14.0-503.el9.x86_64 2024-09-21T09:32:44.663 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum install -y kernel 2024-09-21T09:32:48.104 INFO:teuthology.orchestra.run.smithi092.stdout:CentOS Stream 9 - BaseOS 4.9 MB/s | 8.3 MB 00:01 2024-09-21T09:32:51.288 INFO:teuthology.orchestra.run.smithi092.stdout:CentOS Stream 9 - AppStream 9.8 MB/s | 20 MB 00:02 2024-09-21T09:32:55.555 INFO:teuthology.orchestra.run.smithi092.stdout:CentOS Stream 9 - CRB 14 MB/s | 6.5 MB 00:00 2024-09-21T09:32:57.006 INFO:teuthology.orchestra.run.smithi092.stdout:CentOS Stream 9 - Extras packages 81 kB/s | 19 kB 00:00 2024-09-21T09:32:57.743 INFO:teuthology.orchestra.run.smithi092.stdout:Extra Packages for Enterprise Linux 37 MB/s | 23 MB 00:00 2024-09-21T09:33:03.223 INFO:teuthology.orchestra.run.smithi092.stdout:lab-extras 48 kB/s | 1.7 kB 00:00 2024-09-21T09:33:05.113 INFO:teuthology.orchestra.run.smithi092.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-21T09:33:05.113 INFO:teuthology.orchestra.run.smithi092.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-21T09:33:05.210 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. 2024-09-21T09:33:05.220 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:33:05.220 INFO:teuthology.orchestra.run.smithi092.stdout: Package Architecture Version Repository Size 2024-09-21T09:33:05.220 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:33:05.220 INFO:teuthology.orchestra.run.smithi092.stdout:Installing: 2024-09-21T09:33:05.220 INFO:teuthology.orchestra.run.smithi092.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-21T09:33:05.220 INFO:teuthology.orchestra.run.smithi092.stdout:Installing dependencies: 2024-09-21T09:33:05.221 INFO:teuthology.orchestra.run.smithi092.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-21T09:33:05.221 INFO:teuthology.orchestra.run.smithi092.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-21T09:33:05.221 INFO:teuthology.orchestra.run.smithi092.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-21T09:33:05.221 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:33:05.221 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction Summary 2024-09-21T09:33:05.221 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:33:05.221 INFO:teuthology.orchestra.run.smithi092.stdout:Install 4 Packages 2024-09-21T09:33:05.221 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:33:05.222 INFO:teuthology.orchestra.run.smithi092.stdout:Total download size: 79 M 2024-09-21T09:33:05.222 INFO:teuthology.orchestra.run.smithi092.stdout:Installed size: 126 M 2024-09-21T09:33:05.222 INFO:teuthology.orchestra.run.smithi092.stdout:Downloading Packages: 2024-09-21T09:33:05.382 INFO:teuthology.orchestra.run.smithi092.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 312 kB/s | 36 kB 00:00 2024-09-21T09:33:06.183 INFO:teuthology.orchestra.run.smithi092.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 17 MB/s | 16 MB 00:00 2024-09-21T09:33:06.683 INFO:teuthology.orchestra.run.smithi092.stdout:(3/4): kernel-modules-core-5.14.0-508.el9.x86_6 22 MB/s | 29 MB 00:01 2024-09-21T09:33:07.050 INFO:teuthology.orchestra.run.smithi092.stdout:(4/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 19 MB/s | 35 MB 00:01 2024-09-21T09:33:07.050 INFO:teuthology.orchestra.run.smithi092.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:33:07.050 INFO:teuthology.orchestra.run.smithi092.stdout:Total 43 MB/s | 79 MB 00:01 2024-09-21T09:33:07.731 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction check 2024-09-21T09:33:08.081 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction check succeeded. 2024-09-21T09:33:08.081 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction test 2024-09-21T09:33:09.040 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction test succeeded. 2024-09-21T09:33:09.040 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction 2024-09-21T09:33:11.473 INFO:teuthology.orchestra.run.smithi092.stdout: Preparing : 1/1 2024-09-21T09:33:12.127 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:33:12.208 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:33:14.980 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:33:15.112 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:33:19.935 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:33:20.037 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:33:24.862 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:34:02.837 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:34:02.876 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:34:03.159 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:34:03.159 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:34:03.160 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:34:03.160 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:34:03.934 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:34:03.935 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:34:03.935 INFO:teuthology.orchestra.run.smithi092.stdout:Installed: 2024-09-21T09:34:03.935 INFO:teuthology.orchestra.run.smithi092.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-21T09:34:03.935 INFO:teuthology.orchestra.run.smithi092.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-21T09:34:03.935 INFO:teuthology.orchestra.run.smithi092.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-21T09:34:03.936 INFO:teuthology.orchestra.run.smithi092.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-21T09:34:03.936 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:34:03.936 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! 2024-09-21T09:34:04.259 DEBUG:teuthology.orchestra.run.smithi092:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:34:04.345 INFO:teuthology.orchestra.run.smithi092.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:34:04.345 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:34:04.345 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-21T09:34:04.345 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-21T09:34:04.346 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-21T09:34:04.346 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-21T09:34:04.346 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi092.front.sepia.ceph.com, path=None, version=distro) 2024-09-21T09:34:04.346 DEBUG:teuthology.orchestra.run.smithi092:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:34:04.404 INFO:teuthology.orchestra.run.smithi092.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:34:04.405 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:34:04.405 DEBUG:teuthology.orchestra.run.smithi092:> sudo rpm -qi grub2-tools 2024-09-21T09:34:04.446 INFO:teuthology.orchestra.run.smithi092.stdout:Name : grub2-tools 2024-09-21T09:34:04.446 INFO:teuthology.orchestra.run.smithi092.stdout:Epoch : 1 2024-09-21T09:34:04.446 INFO:teuthology.orchestra.run.smithi092.stdout:Version : 2.06 2024-09-21T09:34:04.446 INFO:teuthology.orchestra.run.smithi092.stdout:Release : 82.el9 2024-09-21T09:34:04.447 INFO:teuthology.orchestra.run.smithi092.stdout:Architecture: x86_64 2024-09-21T09:34:04.447 INFO:teuthology.orchestra.run.smithi092.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-21T09:34:04.447 INFO:teuthology.orchestra.run.smithi092.stdout:Group : Unspecified 2024-09-21T09:34:04.447 INFO:teuthology.orchestra.run.smithi092.stdout:Size : 8274670 2024-09-21T09:34:04.447 INFO:teuthology.orchestra.run.smithi092.stdout:License : GPLv3+ 2024-09-21T09:34:04.447 INFO:teuthology.orchestra.run.smithi092.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-21T09:34:04.447 INFO:teuthology.orchestra.run.smithi092.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-21T09:34:04.447 INFO:teuthology.orchestra.run.smithi092.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-21T09:34:04.447 INFO:teuthology.orchestra.run.smithi092.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-21T09:34:04.447 INFO:teuthology.orchestra.run.smithi092.stdout:Packager : builder@centos.org 2024-09-21T09:34:04.447 INFO:teuthology.orchestra.run.smithi092.stdout:Vendor : CentOS 2024-09-21T09:34:04.447 INFO:teuthology.orchestra.run.smithi092.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-21T09:34:04.447 INFO:teuthology.orchestra.run.smithi092.stdout:Summary : Support tools for GRUB. 2024-09-21T09:34:04.447 INFO:teuthology.orchestra.run.smithi092.stdout:Description : 2024-09-21T09:34:04.448 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:34:04.448 INFO:teuthology.orchestra.run.smithi092.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-21T09:34:04.448 INFO:teuthology.orchestra.run.smithi092.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-21T09:34:04.448 INFO:teuthology.orchestra.run.smithi092.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-21T09:34:04.448 INFO:teuthology.orchestra.run.smithi092.stdout:hardware devices. 2024-09-21T09:34:04.448 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:34:04.448 INFO:teuthology.orchestra.run.smithi092.stdout:This subpackage provides tools for support of all platforms. 2024-09-21T09:34:04.449 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-21T09:34:04.449 INFO:teuthology.task.kernel:Updating grub on smithi092 to boot 5.14.0-508.el9.x86_64 2024-09-21T09:34:04.449 DEBUG:teuthology.orchestra.run.smithi092:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-21T09:34:05.067 INFO:teuthology.orchestra.run.smithi092.stderr:Generating grub configuration file ... 2024-09-21T09:34:06.455 INFO:teuthology.orchestra.run.smithi092.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-21T09:34:06.493 INFO:teuthology.orchestra.run.smithi092.stderr:done 2024-09-21T09:34:06.495 DEBUG:teuthology.orchestra.run.smithi092:> mktemp 2024-09-21T09:34:06.514 INFO:teuthology.orchestra.run.smithi092.stdout:/tmp/tmp.hCAZaqDiX6 2024-09-21T09:34:06.514 DEBUG:teuthology.orchestra.run.smithi092:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.hCAZaqDiX6 2024-09-21T09:34:06.588 DEBUG:teuthology.orchestra.run.smithi092:> sudo chmod 0666 /tmp/tmp.hCAZaqDiX6 2024-09-21T09:34:06.725 DEBUG:teuthology.orchestra.remote:smithi092:/tmp/tmp.hCAZaqDiX6 is 6KB 2024-09-21T09:34:06.774 DEBUG:teuthology.orchestra.run.smithi092:> rm -fr /tmp/tmp.hCAZaqDiX6 2024-09-21T09:34:06.788 DEBUG:teuthology.orchestra.run.smithi092:> sudo /bin/ls /boot/loader/entries || true 2024-09-21T09:34:06.851 INFO:teuthology.orchestra.run.smithi092.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-21T09:34:06.851 INFO:teuthology.orchestra.run.smithi092.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-21T09:34:06.851 INFO:teuthology.orchestra.run.smithi092.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-21T09:34:06.851 INFO:teuthology.orchestra.run.smithi092.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-21T09:34:06.853 DEBUG:teuthology.orchestra.run.smithi092:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-21T09:34:07.011 DEBUG:teuthology.orchestra.run.smithi092:> sudo shutdown -r now 2024-09-21T09:34:37.042 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-21T09:34:37.043 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi092.front.sepia.ceph.com' 2024-09-21T09:34:37.043 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi092.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:35:37.103 DEBUG:teuthology.orchestra.remote:timed out 2024-09-21T09:35:46.107 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi092.front.sepia.ceph.com' 2024-09-21T09:35:46.108 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi092.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:35:46.510 DEBUG:teuthology.orchestra.run.smithi092:> true 2024-09-21T09:35:46.869 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi092.front.sepia.ceph.com' 2024-09-21T09:35:46.869 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-508.el9.x86_64"... 2024-09-21T09:35:46.869 DEBUG:teuthology.orchestra.run.smithi092:> uname -r 2024-09-21T09:35:46.883 INFO:teuthology.orchestra.run.smithi092.stdout:5.14.0-508.el9.x86_64 2024-09-21T09:35:46.884 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-21T09:35:46.884 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-21T09:35:46.884 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-21T09:35:47.885 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-21T09:35:47.885 DEBUG:teuthology.orchestra.run.smithi092:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-21T09:35:47.979 INFO:teuthology.orchestra.run.smithi092.stdout:ttyS1 2024-09-21T09:35:47.998 DEBUG:teuthology.parallel:result is None 2024-09-21T09:35:47.999 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-21T09:35:48.005 INFO:teuthology.task.internal:Creating test directory... 2024-09-21T09:35:48.006 DEBUG:teuthology.orchestra.run.smithi092:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-21T09:35:48.020 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-21T09:35:48.040 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-21T09:35:48.073 INFO:teuthology.task.internal:Creating archive directory... 2024-09-21T09:35:48.073 DEBUG:teuthology.orchestra.run.smithi092:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-21T09:35:48.103 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-21T09:35:48.110 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-21T09:35:48.110 DEBUG:teuthology.orchestra.run.smithi092:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-21T09:35:48.171 INFO:teuthology.orchestra.run.smithi092.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:35:48.181 INFO:teuthology.orchestra.run.smithi092.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:35:48.183 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-21T09:35:48.190 INFO:teuthology.task.internal:Configuring sudo... 2024-09-21T09:35:48.190 DEBUG:teuthology.orchestra.run.smithi092:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-21T09:35:48.244 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-21T09:35:48.253 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-21T09:35:48.253 DEBUG:teuthology.orchestra.run.smithi092:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-21T09:35:48.297 DEBUG:teuthology.orchestra.run.smithi092:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:35:48.375 DEBUG:teuthology.orchestra.run.smithi092:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:35:48.416 DEBUG:teuthology.orchestra.run.smithi092:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:35:48.493 DEBUG:teuthology.orchestra.run.smithi092:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:35:48.532 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:35:48.532 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-21T09:35:48.594 DEBUG:teuthology.orchestra.run.smithi092:> sudo service rsyslog restart 2024-09-21T09:35:48.691 INFO:teuthology.orchestra.run.smithi092.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T09:35:49.017 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-21T09:35:49.024 INFO:teuthology.task.internal:Starting timer... 2024-09-21T09:35:49.024 INFO:teuthology.run_tasks:Running task pcp... 2024-09-21T09:35:49.093 INFO:teuthology.run_tasks:Running task selinux... 2024-09-21T09:35:49.125 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-21T09:35:49.125 DEBUG:teuthology.orchestra.run.smithi092:> sudo service auditd rotate 2024-09-21T09:35:49.188 INFO:teuthology.orchestra.run.smithi092.stdout:Rotating logs: 2024-09-21T09:35:49.191 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-21T09:35:49.191 DEBUG:teuthology.orchestra.run.smithi092:> /usr/sbin/getenforce 2024-09-21T09:35:49.215 INFO:teuthology.orchestra.run.smithi092.stdout:Permissive 2024-09-21T09:35:49.215 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi092.front.sepia.ceph.com': 'permissive'} 2024-09-21T09:35:49.215 DEBUG:teuthology.orchestra.run.smithi092:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T09:35:49.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:35:49.278 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-21T09:35:49.278 DEBUG:teuthology.orchestra.run.smithi092:> sudo /usr/sbin/setenforce permissive 2024-09-21T09:35:49.344 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-21T09:35:49.353 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-09-21T09:35:49.355 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-09-21T09:35:49.453 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-21T09:35:49.463 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-21T09:35:49.464 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi092.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-21T09:39:30.781 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi092.front.sepia.ceph.com')] 2024-09-21T09:39:30.782 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi092.front.sepia.ceph.com' 2024-09-21T09:39:30.783 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi092.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:39:30.861 DEBUG:teuthology.orchestra.run.smithi092:> true 2024-09-21T09:39:30.938 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi092.front.sepia.ceph.com' 2024-09-21T09:39:30.938 INFO:teuthology.run_tasks:Running task clock... 2024-09-21T09:39:30.947 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-21T09:39:30.947 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-21T09:39:30.947 DEBUG:teuthology.orchestra.run.smithi092:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T09:39:31.008 INFO:teuthology.orchestra.run.smithi092.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-21T09:39:31.022 INFO:teuthology.orchestra.run.smithi092.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-21T09:39:31.047 INFO:teuthology.orchestra.run.smithi092.stderr:sudo: ntpd: command not found 2024-09-21T09:39:31.058 INFO:teuthology.orchestra.run.smithi092.stdout:506 Cannot talk to daemon 2024-09-21T09:39:31.073 INFO:teuthology.orchestra.run.smithi092.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-21T09:39:31.086 INFO:teuthology.orchestra.run.smithi092.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-21T09:39:31.144 INFO:teuthology.orchestra.run.smithi092.stderr:bash: line 1: ntpq: command not found 2024-09-21T09:39:31.148 INFO:teuthology.orchestra.run.smithi092.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T09:39:31.148 INFO:teuthology.orchestra.run.smithi092.stdout:=============================================================================== 2024-09-21T09:39:31.148 INFO:teuthology.orchestra.run.smithi092.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:39:31.148 INFO:teuthology.orchestra.run.smithi092.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:39:31.149 INFO:teuthology.orchestra.run.smithi092.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:39:31.149 INFO:teuthology.orchestra.run.smithi092.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:39:31.149 INFO:teuthology.run_tasks:Running task install... 2024-09-21T09:39:31.156 DEBUG:teuthology.task.install:project ceph 2024-09-21T09:39:31.156 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}} 2024-09-21T09:39:31.156 DEBUG:teuthology.task.install:config {'branch': 'squid', 'exclude_packages': ['librados3', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'cephadm', 'ceph-volume'], 'extra_packages': ['librados2'], 'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'} 2024-09-21T09:39:31.156 INFO:teuthology.task.install:Using flavor: default 2024-09-21T09:39:31.161 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-21T09:39:31.162 INFO:teuthology.task.install:extra packages: ['librados2'] 2024-09-21T09:39:31.162 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'squid', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['librados3', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'cephadm', 'ceph-volume'], 'extra_packages': ['librados2'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208', 'tag': None, 'wait_for_package': False} 2024-09-21T09:39:31.162 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:39:31.162 INFO:teuthology.packaging:ref: None 2024-09-21T09:39:31.162 INFO:teuthology.packaging:tag: None 2024-09-21T09:39:31.162 INFO:teuthology.packaging:branch: squid 2024-09-21T09:39:31.162 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:39:31.163 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-09-21T09:39:31.395 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/squid/4e45e6b490e6cbe082cdf155fea780438ec53fc7/centos/9/flavors/default/ 2024-09-21T09:39:31.395 INFO:teuthology.task.install.rpm:Package version is 19.1.1-373.g4e45e6b4 2024-09-21T09:39:31.504 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/squid/4e45e6b490e6cbe082cdf155fea780438ec53fc7/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/squid/4e45e6b490e6cbe082cdf155fea780438ec53fc7/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/squid/4e45e6b490e6cbe082cdf155fea780438ec53fc7/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-21T09:39:31.504 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:39:31.504 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-21T09:39:31.530 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, ceph-immutable-object-cache, ceph-mgr, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, librados2 on remote rpm x86_64 2024-09-21T09:39:31.530 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:39:31.531 INFO:teuthology.packaging:ref: None 2024-09-21T09:39:31.531 INFO:teuthology.packaging:tag: None 2024-09-21T09:39:31.531 INFO:teuthology.packaging:branch: squid 2024-09-21T09:39:31.531 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:39:31.531 DEBUG:teuthology.orchestra.run.smithi092:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/squid/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-21T09:39:31.595 DEBUG:teuthology.orchestra.run.smithi092:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-21T09:39:31.670 DEBUG:teuthology.orchestra.run.smithi092:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-21T09:39:31.734 INFO:teuthology.orchestra.run.smithi092.stdout:check_obsoletes = 1 2024-09-21T09:39:31.736 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum clean all 2024-09-21T09:39:31.975 INFO:teuthology.orchestra.run.smithi092.stdout:45 files removed 2024-09-21T09:39:32.004 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum -y install ceph-radosgw 2024-09-21T09:39:32.546 INFO:teuthology.orchestra.run.smithi092.stdout:ceph packages for x86_64 322 kB/s | 84 kB 00:00 2024-09-21T09:39:32.813 INFO:teuthology.orchestra.run.smithi092.stdout:ceph noarch packages 52 kB/s | 12 kB 00:00 2024-09-21T09:39:33.080 INFO:teuthology.orchestra.run.smithi092.stdout:ceph source packages 8.1 kB/s | 1.9 kB 00:00 2024-09-21T09:39:34.755 INFO:teuthology.orchestra.run.smithi092.stdout:CentOS Stream 9 - BaseOS 5.0 MB/s | 8.3 MB 00:01 2024-09-21T09:39:36.514 INFO:teuthology.orchestra.run.smithi092.stdout:CentOS Stream 9 - AppStream 29 MB/s | 20 MB 00:00 2024-09-21T09:39:41.923 INFO:teuthology.orchestra.run.smithi092.stdout:CentOS Stream 9 - CRB 4.0 MB/s | 6.5 MB 00:01 2024-09-21T09:39:43.624 INFO:teuthology.orchestra.run.smithi092.stdout:CentOS Stream 9 - Extras packages 42 kB/s | 19 kB 00:00 2024-09-21T09:39:44.488 INFO:teuthology.orchestra.run.smithi092.stdout:Extra Packages for Enterprise Linux 31 MB/s | 23 MB 00:00 2024-09-21T09:39:49.977 INFO:teuthology.orchestra.run.smithi092.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-09-21T09:39:52.213 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. 2024-09-21T09:39:52.221 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:39:52.221 INFO:teuthology.orchestra.run.smithi092.stdout: Package Arch Version Repository Size 2024-09-21T09:39:52.221 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:39:52.221 INFO:teuthology.orchestra.run.smithi092.stdout:Installing: 2024-09-21T09:39:52.221 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-radosgw x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 11 M 2024-09-21T09:39:52.221 INFO:teuthology.orchestra.run.smithi092.stdout:Upgrading: 2024-09-21T09:39:52.222 INFO:teuthology.orchestra.run.smithi092.stdout: librados2 x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 3.4 M 2024-09-21T09:39:52.222 INFO:teuthology.orchestra.run.smithi092.stdout: librbd1 x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 3.2 M 2024-09-21T09:39:52.222 INFO:teuthology.orchestra.run.smithi092.stdout:Installing dependencies: 2024-09-21T09:39:52.222 INFO:teuthology.orchestra.run.smithi092.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-09-21T09:39:52.222 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-base x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 5.4 M 2024-09-21T09:39:52.222 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-common x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 21 M 2024-09-21T09:39:52.222 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-selinux x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 25 k 2024-09-21T09:39:52.222 INFO:teuthology.orchestra.run.smithi092.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-21T09:39:52.222 INFO:teuthology.orchestra.run.smithi092.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-09-21T09:39:52.222 INFO:teuthology.orchestra.run.smithi092.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-09-21T09:39:52.222 INFO:teuthology.orchestra.run.smithi092.stdout: libcephfs2 x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 751 k 2024-09-21T09:39:52.222 INFO:teuthology.orchestra.run.smithi092.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-21T09:39:52.222 INFO:teuthology.orchestra.run.smithi092.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-21T09:39:52.223 INFO:teuthology.orchestra.run.smithi092.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-21T09:39:52.223 INFO:teuthology.orchestra.run.smithi092.stdout: libradosstriper1 x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 507 k 2024-09-21T09:39:52.223 INFO:teuthology.orchestra.run.smithi092.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-21T09:39:52.223 INFO:teuthology.orchestra.run.smithi092.stdout: librgw2 x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 5.3 M 2024-09-21T09:39:52.223 INFO:teuthology.orchestra.run.smithi092.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-21T09:39:52.223 INFO:teuthology.orchestra.run.smithi092.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-21T09:39:52.223 INFO:teuthology.orchestra.run.smithi092.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-09-21T09:39:52.223 INFO:teuthology.orchestra.run.smithi092.stdout: python3-ceph-argparse x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 45 k 2024-09-21T09:39:52.223 INFO:teuthology.orchestra.run.smithi092.stdout: python3-ceph-common x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 136 k 2024-09-21T09:39:52.223 INFO:teuthology.orchestra.run.smithi092.stdout: python3-cephfs x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 162 k 2024-09-21T09:39:52.223 INFO:teuthology.orchestra.run.smithi092.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-21T09:39:52.223 INFO:teuthology.orchestra.run.smithi092.stdout: python3-rados x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 321 k 2024-09-21T09:39:52.223 INFO:teuthology.orchestra.run.smithi092.stdout: python3-rbd x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 300 k 2024-09-21T09:39:52.223 INFO:teuthology.orchestra.run.smithi092.stdout: python3-rgw x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 99 k 2024-09-21T09:39:52.224 INFO:teuthology.orchestra.run.smithi092.stdout: qatlib x86_64 24.02.0-1.el9 appstream 221 k 2024-09-21T09:39:52.224 INFO:teuthology.orchestra.run.smithi092.stdout: qatzip-libs x86_64 1.2.0-1.el9 appstream 47 k 2024-09-21T09:39:52.224 INFO:teuthology.orchestra.run.smithi092.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2024-09-21T09:39:52.224 INFO:teuthology.orchestra.run.smithi092.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-21T09:39:52.224 INFO:teuthology.orchestra.run.smithi092.stdout:Installing weak dependencies: 2024-09-21T09:39:52.224 INFO:teuthology.orchestra.run.smithi092.stdout: qatlib-service x86_64 24.02.0-1.el9 appstream 36 k 2024-09-21T09:39:52.224 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:39:52.224 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction Summary 2024-09-21T09:39:52.224 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:39:52.224 INFO:teuthology.orchestra.run.smithi092.stdout:Install 30 Packages 2024-09-21T09:39:52.224 INFO:teuthology.orchestra.run.smithi092.stdout:Upgrade 2 Packages 2024-09-21T09:39:52.224 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:39:52.225 INFO:teuthology.orchestra.run.smithi092.stdout:Total download size: 60 M 2024-09-21T09:39:52.225 INFO:teuthology.orchestra.run.smithi092.stdout:Downloading Packages: 2024-09-21T09:39:53.070 INFO:teuthology.orchestra.run.smithi092.stdout:(1/32): ceph-base-19.1.1-373.g4e45e6b4.el9.x86_ 10 MB/s | 5.4 MB 00:00 2024-09-21T09:39:53.120 INFO:teuthology.orchestra.run.smithi092.stdout:(2/32): ceph-selinux-19.1.1-373.g4e45e6b4.el9.x 505 kB/s | 25 kB 00:00 2024-09-21T09:39:53.228 INFO:teuthology.orchestra.run.smithi092.stdout:(3/32): libcephfs2-19.1.1-373.g4e45e6b4.el9.x86 6.8 MB/s | 751 kB 00:00 2024-09-21T09:39:53.312 INFO:teuthology.orchestra.run.smithi092.stdout:(4/32): libradosstriper1-19.1.1-373.g4e45e6b4.e 6.0 MB/s | 507 kB 00:00 2024-09-21T09:39:53.420 INFO:teuthology.orchestra.run.smithi092.stdout:(5/32): ceph-radosgw-19.1.1-373.g4e45e6b4.el9.x 12 MB/s | 11 MB 00:00 2024-09-21T09:39:53.479 INFO:teuthology.orchestra.run.smithi092.stdout:(6/32): python3-ceph-argparse-19.1.1-373.g4e45e 776 kB/s | 45 kB 00:00 2024-09-21T09:39:53.537 INFO:teuthology.orchestra.run.smithi092.stdout:(7/32): python3-ceph-common-19.1.1-373.g4e45e6b 2.3 MB/s | 136 kB 00:00 2024-09-21T09:39:53.605 INFO:teuthology.orchestra.run.smithi092.stdout:(8/32): python3-cephfs-19.1.1-373.g4e45e6b4.el9 2.4 MB/s | 162 kB 00:00 2024-09-21T09:39:53.671 INFO:teuthology.orchestra.run.smithi092.stdout:(9/32): librgw2-19.1.1-373.g4e45e6b4.el9.x86_64 15 MB/s | 5.3 MB 00:00 2024-09-21T09:39:53.696 INFO:teuthology.orchestra.run.smithi092.stdout:(10/32): python3-rados-19.1.1-373.g4e45e6b4.el9 3.4 MB/s | 321 kB 00:00 2024-09-21T09:39:53.722 INFO:teuthology.orchestra.run.smithi092.stdout:(11/32): python3-rbd-19.1.1-373.g4e45e6b4.el9.x 5.8 MB/s | 300 kB 00:00 2024-09-21T09:39:53.747 INFO:teuthology.orchestra.run.smithi092.stdout:(12/32): python3-rgw-19.1.1-373.g4e45e6b4.el9.x 1.9 MB/s | 99 kB 00:00 2024-09-21T09:39:53.956 INFO:teuthology.orchestra.run.smithi092.stdout:(13/32): ceph-common-19.1.1-373.g4e45e6b4.el9.x 15 MB/s | 21 MB 00:01 2024-09-21T09:39:53.998 INFO:teuthology.orchestra.run.smithi092.stdout:(14/32): boost-program-options-1.75.0-8.el9.x86 388 kB/s | 107 kB 00:00 2024-09-21T09:39:54.056 INFO:teuthology.orchestra.run.smithi092.stdout:(15/32): libpmemobj-1.12.1-1.el9.x86_64.rpm 519 kB/s | 160 kB 00:00 2024-09-21T09:39:54.073 INFO:teuthology.orchestra.run.smithi092.stdout:(16/32): librabbitmq-0.11.0-7.el9.x86_64.rpm 388 kB/s | 45 kB 00:00 2024-09-21T09:39:54.098 INFO:teuthology.orchestra.run.smithi092.stdout:(17/32): librdkafka-1.6.1-102.el9.x86_64.rpm 6.4 MB/s | 662 kB 00:00 2024-09-21T09:39:54.115 INFO:teuthology.orchestra.run.smithi092.stdout:(18/32): python3-prettytable-0.7.2-27.el9.noarc 1.0 MB/s | 42 kB 00:00 2024-09-21T09:39:54.141 INFO:teuthology.orchestra.run.smithi092.stdout:(19/32): lttng-ust-2.12.0-6.el9.x86_64.rpm 3.4 MB/s | 292 kB 00:00 2024-09-21T09:39:54.158 INFO:teuthology.orchestra.run.smithi092.stdout:(20/32): qatlib-service-24.02.0-1.el9.x86_64.rp 851 kB/s | 36 kB 00:00 2024-09-21T09:39:54.175 INFO:teuthology.orchestra.run.smithi092.stdout:(21/32): qatzip-libs-1.2.0-1.el9.x86_64.rpm 1.4 MB/s | 47 kB 00:00 2024-09-21T09:39:54.200 INFO:teuthology.orchestra.run.smithi092.stdout:(22/32): qatlib-24.02.0-1.el9.x86_64.rpm 2.1 MB/s | 221 kB 00:00 2024-09-21T09:39:54.250 INFO:teuthology.orchestra.run.smithi092.stdout:(23/32): gperftools-libs-2.9.1-3.el9.x86_64.rpm 3.3 MB/s | 308 kB 00:00 2024-09-21T09:39:54.275 INFO:teuthology.orchestra.run.smithi092.stdout:(24/32): liboath-2.6.7-2.el9.x86_64.rpm 2.1 MB/s | 49 kB 00:00 2024-09-21T09:39:54.301 INFO:teuthology.orchestra.run.smithi092.stdout:(25/32): libarrow-doc-9.0.0-11.el9.noarch.rpm 300 kB/s | 26 kB 00:00 2024-09-21T09:39:54.326 INFO:teuthology.orchestra.run.smithi092.stdout:(26/32): libunwind-1.6.2-1.el9.x86_64.rpm 1.3 MB/s | 67 kB 00:00 2024-09-21T09:39:54.360 INFO:teuthology.orchestra.run.smithi092.stdout:(27/32): re2-20211101-20.el9.x86_64.rpm 5.6 MB/s | 191 kB 00:00 2024-09-21T09:39:54.410 INFO:teuthology.orchestra.run.smithi092.stdout:(28/32): parquet-libs-9.0.0-11.el9.x86_64.rpm 7.5 MB/s | 839 kB 00:00 2024-09-21T09:39:54.460 INFO:teuthology.orchestra.run.smithi092.stdout:(29/32): thrift-0.15.0-2.el9.x86_64.rpm 16 MB/s | 1.6 MB 00:00 2024-09-21T09:39:54.544 INFO:teuthology.orchestra.run.smithi092.stdout:(30/32): libarrow-9.0.0-11.el9.x86_64.rpm 12 MB/s | 4.4 MB 00:00 2024-09-21T09:39:54.677 INFO:teuthology.orchestra.run.smithi092.stdout:(31/32): librados2-19.1.1-373.g4e45e6b4.el9.x86 13 MB/s | 3.4 MB 00:00 2024-09-21T09:39:54.736 INFO:teuthology.orchestra.run.smithi092.stdout:(32/32): librbd1-19.1.1-373.g4e45e6b4.el9.x86_6 11 MB/s | 3.2 MB 00:00 2024-09-21T09:39:54.740 INFO:teuthology.orchestra.run.smithi092.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:39:54.740 INFO:teuthology.orchestra.run.smithi092.stdout:Total 24 MB/s | 60 MB 00:02 2024-09-21T09:39:54.837 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction check 2024-09-21T09:39:54.872 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction check succeeded. 2024-09-21T09:39:54.872 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction test 2024-09-21T09:39:55.551 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction test succeeded. 2024-09-21T09:39:55.551 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction 2024-09-21T09:39:57.052 INFO:teuthology.orchestra.run.smithi092.stdout: Preparing : 1/1 2024-09-21T09:39:57.133 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/34 2024-09-21T09:39:57.165 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/34 2024-09-21T09:39:57.191 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/34 2024-09-21T09:39:57.213 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : qatlib-service-24.02.0-1.el9.x86_64 4/34 2024-09-21T09:39:57.239 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: qatlib-service-24.02.0-1.el9.x86_64 4/34 2024-09-21T09:39:57.433 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: qatlib-24.02.0-1.el9.x86_64 5/34 2024-09-21T09:39:57.486 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : qatlib-24.02.0-1.el9.x86_64 5/34 2024-09-21T09:39:57.874 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : thrift-0.15.0-2.el9.x86_64 6/34 2024-09-21T09:39:57.942 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : librados2-2:19.1.1-373.g4e45e6b4.el9.x86_64 7/34 2024-09-21T09:39:58.027 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: librados2-2:19.1.1-373.g4e45e6b4.el9.x86_64 7/34 2024-09-21T09:39:58.116 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-rados-2:19.1.1-373.g4e45e6b4.el9.x86_64 8/34 2024-09-21T09:39:58.144 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : libcephfs2-2:19.1.1-373.g4e45e6b4.el9.x86_64 9/34 2024-09-21T09:39:58.190 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: libcephfs2-2:19.1.1-373.g4e45e6b4.el9.x86_64 9/34 2024-09-21T09:39:58.227 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : liboath-2.6.7-2.el9.x86_64 10/34 2024-09-21T09:39:58.271 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-ceph-argparse-2:19.1.1-373.g4e45e6b4.el9.x 11/34 2024-09-21T09:39:58.333 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-cephfs-2:19.1.1-373.g4e45e6b4.el9.x86_64 12/34 2024-09-21T09:39:58.363 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : libradosstriper1-2:19.1.1-373.g4e45e6b4.el9.x86_64 13/34 2024-09-21T09:39:58.427 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: libradosstriper1-2:19.1.1-373.g4e45e6b4.el9.x86_64 13/34 2024-09-21T09:39:58.461 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : qatzip-libs-1.2.0-1.el9.x86_64 14/34 2024-09-21T09:39:58.571 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : re2-1:20211101-20.el9.x86_64 15/34 2024-09-21T09:39:58.604 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 16/34 2024-09-21T09:39:58.654 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 17/34 2024-09-21T09:39:58.694 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 18/34 2024-09-21T09:39:58.723 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 19/34 2024-09-21T09:39:58.756 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 20/34 2024-09-21T09:39:59.075 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 21/34 2024-09-21T09:39:59.118 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : librbd1-2:19.1.1-373.g4e45e6b4.el9.x86_64 22/34 2024-09-21T09:39:59.209 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: librbd1-2:19.1.1-373.g4e45e6b4.el9.x86_64 22/34 2024-09-21T09:39:59.239 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-rbd-2:19.1.1-373.g4e45e6b4.el9.x86_64 23/34 2024-09-21T09:39:59.275 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 24/34 2024-09-21T09:39:59.759 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 25/34 2024-09-21T09:39:59.795 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : librgw2-2:19.1.1-373.g4e45e6b4.el9.x86_64 26/34 2024-09-21T09:39:59.850 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: librgw2-2:19.1.1-373.g4e45e6b4.el9.x86_64 26/34 2024-09-21T09:39:59.920 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-rgw-2:19.1.1-373.g4e45e6b4.el9.x86_64 27/34 2024-09-21T09:39:59.956 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-ceph-common-2:19.1.1-373.g4e45e6b4.el9.x86 28/34 2024-09-21T09:40:02.309 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-common-2:19.1.1-373.g4e45e6b4.el9.x86_64 29/34 2024-09-21T09:40:02.344 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : ceph-common-2:19.1.1-373.g4e45e6b4.el9.x86_64 29/34 2024-09-21T09:40:02.907 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-common-2:19.1.1-373.g4e45e6b4.el9.x86_64 29/34 2024-09-21T09:40:02.955 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : ceph-base-2:19.1.1-373.g4e45e6b4.el9.x86_64 30/34 2024-09-21T09:40:03.046 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-base-2:19.1.1-373.g4e45e6b4.el9.x86_64 30/34 2024-09-21T09:40:03.046 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-21T09:40:03.047 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-21T09:40:03.047 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:03.082 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : ceph-selinux-2:19.1.1-373.g4e45e6b4.el9.x86_64 31/34 2024-09-21T09:40:13.339 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-selinux-2:19.1.1-373.g4e45e6b4.el9.x86_64 31/34 2024-09-21T09:40:13.339 INFO:teuthology.orchestra.run.smithi092.stdout:skipping the directory /sys 2024-09-21T09:40:13.340 INFO:teuthology.orchestra.run.smithi092.stdout:skipping the directory /proc 2024-09-21T09:40:13.340 INFO:teuthology.orchestra.run.smithi092.stdout:skipping the directory /mnt 2024-09-21T09:40:13.340 INFO:teuthology.orchestra.run.smithi092.stdout:skipping the directory /var/tmp 2024-09-21T09:40:13.340 INFO:teuthology.orchestra.run.smithi092.stdout:skipping the directory /home 2024-09-21T09:40:13.340 INFO:teuthology.orchestra.run.smithi092.stdout:skipping the directory /root 2024-09-21T09:40:13.340 INFO:teuthology.orchestra.run.smithi092.stdout:skipping the directory /tmp 2024-09-21T09:40:13.340 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:14.330 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : ceph-radosgw-2:19.1.1-373.g4e45e6b4.el9.x86_64 32/34 2024-09-21T09:40:14.355 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-radosgw-2:19.1.1-373.g4e45e6b4.el9.x86_64 32/34 2024-09-21T09:40:14.356 INFO:teuthology.orchestra.run.smithi092.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:40:14.356 INFO:teuthology.orchestra.run.smithi092.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-21T09:40:14.356 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:40:14.356 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:40:14.356 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:14.357 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 33/34 2024-09-21T09:40:14.441 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 33/34 2024-09-21T09:40:14.441 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 34/34 2024-09-21T09:40:15.609 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 34/34 2024-09-21T09:40:15.609 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-base-2:19.1.1-373.g4e45e6b4.el9.x86_64 1/34 2024-09-21T09:40:15.609 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-common-2:19.1.1-373.g4e45e6b4.el9.x86_64 2/34 2024-09-21T09:40:15.609 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-radosgw-2:19.1.1-373.g4e45e6b4.el9.x86_64 3/34 2024-09-21T09:40:15.610 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-selinux-2:19.1.1-373.g4e45e6b4.el9.x86_64 4/34 2024-09-21T09:40:15.610 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : libcephfs2-2:19.1.1-373.g4e45e6b4.el9.x86_64 5/34 2024-09-21T09:40:15.610 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : libradosstriper1-2:19.1.1-373.g4e45e6b4.el9.x86_64 6/34 2024-09-21T09:40:15.610 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : librgw2-2:19.1.1-373.g4e45e6b4.el9.x86_64 7/34 2024-09-21T09:40:15.610 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-ceph-argparse-2:19.1.1-373.g4e45e6b4.el9.x 8/34 2024-09-21T09:40:15.610 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-ceph-common-2:19.1.1-373.g4e45e6b4.el9.x86 9/34 2024-09-21T09:40:15.610 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-cephfs-2:19.1.1-373.g4e45e6b4.el9.x86_64 10/34 2024-09-21T09:40:15.610 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-rados-2:19.1.1-373.g4e45e6b4.el9.x86_64 11/34 2024-09-21T09:40:15.610 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-rbd-2:19.1.1-373.g4e45e6b4.el9.x86_64 12/34 2024-09-21T09:40:15.610 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-rgw-2:19.1.1-373.g4e45e6b4.el9.x86_64 13/34 2024-09-21T09:40:15.610 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/34 2024-09-21T09:40:15.610 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/34 2024-09-21T09:40:15.611 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/34 2024-09-21T09:40:15.611 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/34 2024-09-21T09:40:15.611 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/34 2024-09-21T09:40:15.611 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/34 2024-09-21T09:40:15.611 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : qatlib-24.02.0-1.el9.x86_64 20/34 2024-09-21T09:40:15.611 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : qatlib-service-24.02.0-1.el9.x86_64 21/34 2024-09-21T09:40:15.611 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : qatzip-libs-1.2.0-1.el9.x86_64 22/34 2024-09-21T09:40:15.611 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 23/34 2024-09-21T09:40:15.611 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 24/34 2024-09-21T09:40:15.611 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 25/34 2024-09-21T09:40:15.611 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 26/34 2024-09-21T09:40:15.611 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 27/34 2024-09-21T09:40:15.611 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 28/34 2024-09-21T09:40:15.612 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : re2-1:20211101-20.el9.x86_64 29/34 2024-09-21T09:40:15.612 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 30/34 2024-09-21T09:40:15.612 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : librados2-2:19.1.1-373.g4e45e6b4.el9.x86_64 31/34 2024-09-21T09:40:15.612 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 32/34 2024-09-21T09:40:15.612 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : librbd1-2:19.1.1-373.g4e45e6b4.el9.x86_64 33/34 2024-09-21T09:40:16.677 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 34/34 2024-09-21T09:40:16.677 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:16.677 INFO:teuthology.orchestra.run.smithi092.stdout:Upgraded: 2024-09-21T09:40:16.677 INFO:teuthology.orchestra.run.smithi092.stdout: librados2-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:16.678 INFO:teuthology.orchestra.run.smithi092.stdout: librbd1-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:16.678 INFO:teuthology.orchestra.run.smithi092.stdout:Installed: 2024-09-21T09:40:16.678 INFO:teuthology.orchestra.run.smithi092.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-09-21T09:40:16.678 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-base-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:16.678 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-common-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:16.678 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-radosgw-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:16.679 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-selinux-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:16.679 INFO:teuthology.orchestra.run.smithi092.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-21T09:40:16.679 INFO:teuthology.orchestra.run.smithi092.stdout: libarrow-9.0.0-11.el9.x86_64 2024-09-21T09:40:16.679 INFO:teuthology.orchestra.run.smithi092.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-09-21T09:40:16.679 INFO:teuthology.orchestra.run.smithi092.stdout: libcephfs2-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:16.679 INFO:teuthology.orchestra.run.smithi092.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-21T09:40:16.679 INFO:teuthology.orchestra.run.smithi092.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-21T09:40:16.680 INFO:teuthology.orchestra.run.smithi092.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-21T09:40:16.680 INFO:teuthology.orchestra.run.smithi092.stdout: libradosstriper1-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:16.680 INFO:teuthology.orchestra.run.smithi092.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-21T09:40:16.680 INFO:teuthology.orchestra.run.smithi092.stdout: librgw2-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:16.680 INFO:teuthology.orchestra.run.smithi092.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-21T09:40:16.680 INFO:teuthology.orchestra.run.smithi092.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-21T09:40:16.681 INFO:teuthology.orchestra.run.smithi092.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-09-21T09:40:16.681 INFO:teuthology.orchestra.run.smithi092.stdout: python3-ceph-argparse-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:16.681 INFO:teuthology.orchestra.run.smithi092.stdout: python3-ceph-common-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:16.681 INFO:teuthology.orchestra.run.smithi092.stdout: python3-cephfs-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:16.681 INFO:teuthology.orchestra.run.smithi092.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-21T09:40:16.681 INFO:teuthology.orchestra.run.smithi092.stdout: python3-rados-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:16.682 INFO:teuthology.orchestra.run.smithi092.stdout: python3-rbd-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:16.682 INFO:teuthology.orchestra.run.smithi092.stdout: python3-rgw-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:16.682 INFO:teuthology.orchestra.run.smithi092.stdout: qatlib-24.02.0-1.el9.x86_64 2024-09-21T09:40:16.682 INFO:teuthology.orchestra.run.smithi092.stdout: qatlib-service-24.02.0-1.el9.x86_64 2024-09-21T09:40:16.682 INFO:teuthology.orchestra.run.smithi092.stdout: qatzip-libs-1.2.0-1.el9.x86_64 2024-09-21T09:40:16.682 INFO:teuthology.orchestra.run.smithi092.stdout: re2-1:20211101-20.el9.x86_64 2024-09-21T09:40:16.683 INFO:teuthology.orchestra.run.smithi092.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-21T09:40:16.683 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:16.683 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! 2024-09-21T09:40:17.008 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum -y install ceph-test 2024-09-21T09:40:17.589 INFO:teuthology.orchestra.run.smithi092.stdout:Last metadata expiration check: 0:00:28 ago on Sat 21 Sep 2024 09:39:49 AM UTC. 2024-09-21T09:40:17.739 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. 2024-09-21T09:40:17.740 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:40:17.740 INFO:teuthology.orchestra.run.smithi092.stdout: Package Arch Version Repository Size 2024-09-21T09:40:17.740 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:40:17.740 INFO:teuthology.orchestra.run.smithi092.stdout:Installing: 2024-09-21T09:40:17.740 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-test x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 49 M 2024-09-21T09:40:17.740 INFO:teuthology.orchestra.run.smithi092.stdout:Installing dependencies: 2024-09-21T09:40:17.740 INFO:teuthology.orchestra.run.smithi092.stdout: libcephsqlite x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 163 k 2024-09-21T09:40:17.740 INFO:teuthology.orchestra.run.smithi092.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-21T09:40:17.740 INFO:teuthology.orchestra.run.smithi092.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-21T09:40:17.740 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:17.741 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction Summary 2024-09-21T09:40:17.741 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:40:17.741 INFO:teuthology.orchestra.run.smithi092.stdout:Install 4 Packages 2024-09-21T09:40:17.741 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:17.741 INFO:teuthology.orchestra.run.smithi092.stdout:Total download size: 49 M 2024-09-21T09:40:17.741 INFO:teuthology.orchestra.run.smithi092.stdout:Installed size: 209 M 2024-09-21T09:40:17.741 INFO:teuthology.orchestra.run.smithi092.stdout:Downloading Packages: 2024-09-21T09:40:18.145 INFO:teuthology.orchestra.run.smithi092.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 2.4 MB/s | 304 kB 00:00 2024-09-21T09:40:18.187 INFO:teuthology.orchestra.run.smithi092.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.5 MB/s | 64 kB 00:00 2024-09-21T09:40:18.245 INFO:teuthology.orchestra.run.smithi092.stdout:(3/4): libcephsqlite-19.1.1-373.g4e45e6b4.el9.x 722 kB/s | 163 kB 00:00 2024-09-21T09:40:19.570 INFO:teuthology.orchestra.run.smithi092.stdout:(4/4): ceph-test-19.1.1-373.g4e45e6b4.el9.x86_6 31 MB/s | 49 MB 00:01 2024-09-21T09:40:19.572 INFO:teuthology.orchestra.run.smithi092.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:40:19.573 INFO:teuthology.orchestra.run.smithi092.stdout:Total 27 MB/s | 49 MB 00:01 2024-09-21T09:40:19.600 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction check 2024-09-21T09:40:19.620 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction check succeeded. 2024-09-21T09:40:19.620 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction test 2024-09-21T09:40:19.990 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction test succeeded. 2024-09-21T09:40:19.991 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction 2024-09-21T09:40:20.508 INFO:teuthology.orchestra.run.smithi092.stdout: Preparing : 1/1 2024-09-21T09:40:20.598 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-21T09:40:20.644 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-21T09:40:20.672 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : libcephsqlite-2:19.1.1-373.g4e45e6b4.el9.x86_64 3/4 2024-09-21T09:40:25.317 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: libcephsqlite-2:19.1.1-373.g4e45e6b4.el9.x86_64 3/4 2024-09-21T09:40:25.395 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : ceph-test-2:19.1.1-373.g4e45e6b4.el9.x86_64 4/4 2024-09-21T09:40:27.820 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-test-2:19.1.1-373.g4e45e6b4.el9.x86_64 4/4 2024-09-21T09:40:27.820 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-test-2:19.1.1-373.g4e45e6b4.el9.x86_64 1/4 2024-09-21T09:40:27.820 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : libcephsqlite-2:19.1.1-373.g4e45e6b4.el9.x86_64 2/4 2024-09-21T09:40:27.820 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-21T09:40:28.137 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-21T09:40:28.137 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:28.137 INFO:teuthology.orchestra.run.smithi092.stdout:Installed: 2024-09-21T09:40:28.137 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-test-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:28.137 INFO:teuthology.orchestra.run.smithi092.stdout: libcephsqlite-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:28.138 INFO:teuthology.orchestra.run.smithi092.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-21T09:40:28.138 INFO:teuthology.orchestra.run.smithi092.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-21T09:40:28.138 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:28.138 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! 2024-09-21T09:40:28.431 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum -y install ceph 2024-09-21T09:40:29.018 INFO:teuthology.orchestra.run.smithi092.stdout:Last metadata expiration check: 0:00:40 ago on Sat 21 Sep 2024 09:39:49 AM UTC. 2024-09-21T09:40:29.151 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. 2024-09-21T09:40:29.153 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:40:29.153 INFO:teuthology.orchestra.run.smithi092.stdout: Package Arch Version Repository Size 2024-09-21T09:40:29.154 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:40:29.154 INFO:teuthology.orchestra.run.smithi092.stdout:Installing: 2024-09-21T09:40:29.154 INFO:teuthology.orchestra.run.smithi092.stdout: ceph x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 6.6 k 2024-09-21T09:40:29.154 INFO:teuthology.orchestra.run.smithi092.stdout:Installing dependencies: 2024-09-21T09:40:29.154 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mds x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 2.4 M 2024-09-21T09:40:29.154 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mgr x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 1.5 M 2024-09-21T09:40:29.154 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mgr-modules-core noarch 2:19.1.1-373.g4e45e6b4.el9 ceph-noarch 249 k 2024-09-21T09:40:29.154 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mon x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 4.7 M 2024-09-21T09:40:29.154 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-osd x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 17 M 2024-09-21T09:40:29.154 INFO:teuthology.orchestra.run.smithi092.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2024-09-21T09:40:29.154 INFO:teuthology.orchestra.run.smithi092.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-09-21T09:40:29.154 INFO:teuthology.orchestra.run.smithi092.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2024-09-21T09:40:29.154 INFO:teuthology.orchestra.run.smithi092.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-21T09:40:29.154 INFO:teuthology.orchestra.run.smithi092.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-21T09:40:29.155 INFO:teuthology.orchestra.run.smithi092.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-21T09:40:29.155 INFO:teuthology.orchestra.run.smithi092.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-21T09:40:29.155 INFO:teuthology.orchestra.run.smithi092.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-21T09:40:29.155 INFO:teuthology.orchestra.run.smithi092.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-21T09:40:29.155 INFO:teuthology.orchestra.run.smithi092.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-21T09:40:29.155 INFO:teuthology.orchestra.run.smithi092.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-21T09:40:29.155 INFO:teuthology.orchestra.run.smithi092.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-21T09:40:29.155 INFO:teuthology.orchestra.run.smithi092.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-21T09:40:29.155 INFO:teuthology.orchestra.run.smithi092.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-21T09:40:29.155 INFO:teuthology.orchestra.run.smithi092.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-21T09:40:29.155 INFO:teuthology.orchestra.run.smithi092.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-21T09:40:29.155 INFO:teuthology.orchestra.run.smithi092.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-21T09:40:29.155 INFO:teuthology.orchestra.run.smithi092.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-21T09:40:29.155 INFO:teuthology.orchestra.run.smithi092.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-21T09:40:29.156 INFO:teuthology.orchestra.run.smithi092.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-21T09:40:29.156 INFO:teuthology.orchestra.run.smithi092.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-21T09:40:29.156 INFO:teuthology.orchestra.run.smithi092.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-21T09:40:29.156 INFO:teuthology.orchestra.run.smithi092.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-21T09:40:29.156 INFO:teuthology.orchestra.run.smithi092.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-21T09:40:29.156 INFO:teuthology.orchestra.run.smithi092.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-21T09:40:29.156 INFO:teuthology.orchestra.run.smithi092.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-21T09:40:29.156 INFO:teuthology.orchestra.run.smithi092.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-21T09:40:29.156 INFO:teuthology.orchestra.run.smithi092.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-21T09:40:29.156 INFO:teuthology.orchestra.run.smithi092.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-21T09:40:29.156 INFO:teuthology.orchestra.run.smithi092.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-21T09:40:29.156 INFO:teuthology.orchestra.run.smithi092.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-21T09:40:29.156 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:29.156 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction Summary 2024-09-21T09:40:29.156 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:40:29.156 INFO:teuthology.orchestra.run.smithi092.stdout:Install 37 Packages 2024-09-21T09:40:29.157 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:29.157 INFO:teuthology.orchestra.run.smithi092.stdout:Total download size: 30 M 2024-09-21T09:40:29.157 INFO:teuthology.orchestra.run.smithi092.stdout:Installed size: 108 M 2024-09-21T09:40:29.157 INFO:teuthology.orchestra.run.smithi092.stdout:Downloading Packages: 2024-09-21T09:40:29.728 INFO:teuthology.orchestra.run.smithi092.stdout:(1/37): ceph-19.1.1-373.g4e45e6b4.el9.x86_64.rp 46 kB/s | 6.6 kB 00:00 2024-09-21T09:40:29.904 INFO:teuthology.orchestra.run.smithi092.stdout:(2/37): ceph-mgr-19.1.1-373.g4e45e6b4.el9.x86_6 4.8 MB/s | 1.5 MB 00:00 2024-09-21T09:40:29.946 INFO:teuthology.orchestra.run.smithi092.stdout:(3/37): ceph-mds-19.1.1-373.g4e45e6b4.el9.x86_6 6.6 MB/s | 2.4 MB 00:00 2024-09-21T09:40:30.021 INFO:teuthology.orchestra.run.smithi092.stdout:(4/37): ceph-mgr-modules-core-19.1.1-373.g4e45e 3.3 MB/s | 249 kB 00:00 2024-09-21T09:40:30.413 INFO:teuthology.orchestra.run.smithi092.stdout:(5/37): ceph-osd-19.1.1-373.g4e45e6b4.el9.x86_6 33 MB/s | 17 MB 00:00 2024-09-21T09:40:30.521 INFO:teuthology.orchestra.run.smithi092.stdout:(6/37): ceph-mon-19.1.1-373.g4e45e6b4.el9.x86_6 5.9 MB/s | 4.7 MB 00:00 2024-09-21T09:40:30.922 INFO:teuthology.orchestra.run.smithi092.stdout:(7/37): python3-cffi-1.14.5-5.el9.x86_64.rpm 281 kB/s | 253 kB 00:00 2024-09-21T09:40:31.189 INFO:teuthology.orchestra.run.smithi092.stdout:(8/37): python3-ply-3.11-14.el9.noarch.rpm 160 kB/s | 106 kB 00:00 2024-09-21T09:40:31.389 INFO:teuthology.orchestra.run.smithi092.stdout:(9/37): python3-cryptography-36.0.1-4.el9.x86_6 1.3 MB/s | 1.2 MB 00:00 2024-09-21T09:40:31.764 INFO:teuthology.orchestra.run.smithi092.stdout:(10/37): python3-pycparser-2.20-6.el9.noarch.rp 161 kB/s | 135 kB 00:00 2024-09-21T09:40:31.889 INFO:teuthology.orchestra.run.smithi092.stdout:(11/37): lua-5.4.4-4.el9.x86_64.rpm 1.5 MB/s | 188 kB 00:00 2024-09-21T09:40:31.931 INFO:teuthology.orchestra.run.smithi092.stdout:(12/37): python3-mako-1.1.4-6.el9.noarch.rpm 4.1 MB/s | 172 kB 00:00 2024-09-21T09:40:31.973 INFO:teuthology.orchestra.run.smithi092.stdout:(13/37): python3-markupsafe-1.1.1-12.el9.x86_64 836 kB/s | 35 kB 00:00 2024-09-21T09:40:31.998 INFO:teuthology.orchestra.run.smithi092.stdout:(14/37): python3-requests-2.25.1-8.el9.noarch.r 154 kB/s | 125 kB 00:00 2024-09-21T09:40:32.023 INFO:teuthology.orchestra.run.smithi092.stdout:(15/37): python3-packaging-20.9-5.el9.noarch.rp 1.5 MB/s | 77 kB 00:00 2024-09-21T09:40:32.065 INFO:teuthology.orchestra.run.smithi092.stdout:(16/37): python3-toml-0.10.2-6.el9.noarch.rpm 1.0 MB/s | 42 kB 00:00 2024-09-21T09:40:32.090 INFO:teuthology.orchestra.run.smithi092.stdout:(17/37): python3-pytz-2021.1-5.el9.noarch.rpm 556 kB/s | 51 kB 00:00 2024-09-21T09:40:32.132 INFO:teuthology.orchestra.run.smithi092.stdout:(18/37): luarocks-3.9.2-5.el9.noarch.rpm 3.5 MB/s | 151 kB 00:00 2024-09-21T09:40:32.161 INFO:teuthology.orchestra.run.smithi092.stdout:(19/37): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 1.5 MB/s | 43 kB 00:00 2024-09-21T09:40:32.186 INFO:teuthology.orchestra.run.smithi092.stdout:(20/37): lua-devel-5.4.4-4.el9.x86_64.rpm 185 kB/s | 22 kB 00:00 2024-09-21T09:40:32.220 INFO:teuthology.orchestra.run.smithi092.stdout:(21/37): python3-urllib3-1.26.5-6.el9.noarch.rp 258 kB/s | 215 kB 00:00 2024-09-21T09:40:32.245 INFO:teuthology.orchestra.run.smithi092.stdout:(22/37): python3-cheroot-8.6.0-4.el9.noarch.rpm 2.0 MB/s | 172 kB 00:00 2024-09-21T09:40:32.279 INFO:teuthology.orchestra.run.smithi092.stdout:(23/37): python3-jaraco-classes-3.2.1-5.el9.noa 678 kB/s | 18 kB 00:00 2024-09-21T09:40:32.304 INFO:teuthology.orchestra.run.smithi092.stdout:(24/37): python3-jaraco-collections-3.0.0-8.el9 997 kB/s | 23 kB 00:00 2024-09-21T09:40:32.329 INFO:teuthology.orchestra.run.smithi092.stdout:(25/37): python3-jaraco-8.2.1-3.el9.noarch.rpm 99 kB/s | 11 kB 00:00 2024-09-21T09:40:32.355 INFO:teuthology.orchestra.run.smithi092.stdout:(26/37): python3-jaraco-functools-3.5.0-2.el9.n 388 kB/s | 19 kB 00:00 2024-09-21T09:40:32.380 INFO:teuthology.orchestra.run.smithi092.stdout:(27/37): python3-jaraco-text-3.2.0-6.el9.noarch 392 kB/s | 20 kB 00:00 2024-09-21T09:40:32.405 INFO:teuthology.orchestra.run.smithi092.stdout:(28/37): python3-logutils-0.3.5-21.el9.noarch.r 923 kB/s | 46 kB 00:00 2024-09-21T09:40:32.431 INFO:teuthology.orchestra.run.smithi092.stdout:(29/37): python3-cherrypy-18.6.1-2.el9.noarch.r 1.4 MB/s | 358 kB 00:00 2024-09-21T09:40:32.456 INFO:teuthology.orchestra.run.smithi092.stdout:(30/37): python3-more-itertools-8.12.0-2.el9.no 1.0 MB/s | 79 kB 00:00 2024-09-21T09:40:32.481 INFO:teuthology.orchestra.run.smithi092.stdout:(31/37): python3-portend-3.1.0-2.el9.noarch.rpm 328 kB/s | 16 kB 00:00 2024-09-21T09:40:32.506 INFO:teuthology.orchestra.run.smithi092.stdout:(32/37): python3-tempora-5.0.0-2.el9.noarch.rpm 1.4 MB/s | 36 kB 00:00 2024-09-21T09:40:32.532 INFO:teuthology.orchestra.run.smithi092.stdout:(33/37): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.2 MB/s | 90 kB 00:00 2024-09-21T09:40:32.557 INFO:teuthology.orchestra.run.smithi092.stdout:(34/37): python3-pecan-1.4.2-3.el9.noarch.rpm 1.8 MB/s | 272 kB 00:00 2024-09-21T09:40:32.582 INFO:teuthology.orchestra.run.smithi092.stdout:(35/37): python3-zc-lockfile-2.0-10.el9.noarch. 800 kB/s | 20 kB 00:00 2024-09-21T09:40:32.607 INFO:teuthology.orchestra.run.smithi092.stdout:(36/37): python3-webob-1.8.8-2.el9.noarch.rpm 2.2 MB/s | 230 kB 00:00 2024-09-21T09:40:32.632 INFO:teuthology.orchestra.run.smithi092.stdout:(37/37): python3-werkzeug-2.0.3-3.el9.1.noarch. 4.2 MB/s | 427 kB 00:00 2024-09-21T09:40:32.637 INFO:teuthology.orchestra.run.smithi092.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:40:32.637 INFO:teuthology.orchestra.run.smithi092.stdout:Total 8.7 MB/s | 30 MB 00:03 2024-09-21T09:40:32.789 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction check 2024-09-21T09:40:32.829 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction check succeeded. 2024-09-21T09:40:32.829 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction test 2024-09-21T09:40:33.295 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction test succeeded. 2024-09-21T09:40:33.295 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction 2024-09-21T09:40:34.662 INFO:teuthology.orchestra.run.smithi092.stdout: Preparing : 1/1 2024-09-21T09:40:34.719 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/37 2024-09-21T09:40:34.751 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/37 2024-09-21T09:40:34.779 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : lua-5.4.4-4.el9.x86_64 3/37 2024-09-21T09:40:34.851 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 4/37 2024-09-21T09:40:34.895 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : luarocks-3.9.2-5.el9.noarch 5/37 2024-09-21T09:40:34.927 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 6/37 2024-09-21T09:40:35.029 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 7/37 2024-09-21T09:40:35.131 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 8/37 2024-09-21T09:40:35.172 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 9/37 2024-09-21T09:40:35.208 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 10/37 2024-09-21T09:40:35.237 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 11/37 2024-09-21T09:40:35.271 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 12/37 2024-09-21T09:40:35.305 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 13/37 2024-09-21T09:40:35.345 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 14/37 2024-09-21T09:40:35.380 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 15/37 2024-09-21T09:40:35.414 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 16/37 2024-09-21T09:40:35.463 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 17/37 2024-09-21T09:40:35.495 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-packaging-20.9-5.el9.noarch 18/37 2024-09-21T09:40:35.568 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 19/37 2024-09-21T09:40:35.706 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 20/37 2024-09-21T09:40:35.794 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 21/37 2024-09-21T09:40:35.847 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 22/37 2024-09-21T09:40:35.886 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 23/37 2024-09-21T09:40:35.935 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-ply-3.11-14.el9.noarch 24/37 2024-09-21T09:40:35.990 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 25/37 2024-09-21T09:40:36.175 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 26/37 2024-09-21T09:40:36.223 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 27/37 2024-09-21T09:40:36.301 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 28/37 2024-09-21T09:40:36.389 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 29/37 2024-09-21T09:40:36.439 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 30/37 2024-09-21T09:40:36.556 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 31/37 2024-09-21T09:40:36.711 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : ceph-mgr-modules-core-2:19.1.1-373.g4e45e6b4.el9.n 32/37 2024-09-21T09:40:36.740 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : ceph-mgr-2:19.1.1-373.g4e45e6b4.el9.x86_64 33/37 2024-09-21T09:40:36.765 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-mgr-2:19.1.1-373.g4e45e6b4.el9.x86_64 33/37 2024-09-21T09:40:36.765 INFO:teuthology.orchestra.run.smithi092.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:40:36.765 INFO:teuthology.orchestra.run.smithi092.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-21T09:40:36.765 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:40:36.765 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:40:36.765 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:38.216 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : ceph-osd-2:19.1.1-373.g4e45e6b4.el9.x86_64 34/37 2024-09-21T09:40:38.244 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-osd-2:19.1.1-373.g4e45e6b4.el9.x86_64 34/37 2024-09-21T09:40:38.244 INFO:teuthology.orchestra.run.smithi092.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:40:38.244 INFO:teuthology.orchestra.run.smithi092.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-21T09:40:38.244 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:40:38.244 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:40:38.244 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:38.684 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : ceph-mon-2:19.1.1-373.g4e45e6b4.el9.x86_64 35/37 2024-09-21T09:40:38.708 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-mon-2:19.1.1-373.g4e45e6b4.el9.x86_64 35/37 2024-09-21T09:40:38.708 INFO:teuthology.orchestra.run.smithi092.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:40:38.708 INFO:teuthology.orchestra.run.smithi092.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-21T09:40:38.708 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:40:38.708 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:40:38.708 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:38.934 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : ceph-mds-2:19.1.1-373.g4e45e6b4.el9.x86_64 36/37 2024-09-21T09:40:38.958 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-mds-2:19.1.1-373.g4e45e6b4.el9.x86_64 36/37 2024-09-21T09:40:38.958 INFO:teuthology.orchestra.run.smithi092.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:40:38.958 INFO:teuthology.orchestra.run.smithi092.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-21T09:40:38.959 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:40:38.959 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:40:38.959 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:39.023 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : ceph-2:19.1.1-373.g4e45e6b4.el9.x86_64 37/37 2024-09-21T09:40:40.572 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-2:19.1.1-373.g4e45e6b4.el9.x86_64 37/37 2024-09-21T09:40:40.573 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-2:19.1.1-373.g4e45e6b4.el9.x86_64 1/37 2024-09-21T09:40:40.573 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-mds-2:19.1.1-373.g4e45e6b4.el9.x86_64 2/37 2024-09-21T09:40:40.573 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-mgr-2:19.1.1-373.g4e45e6b4.el9.x86_64 3/37 2024-09-21T09:40:40.573 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-mon-2:19.1.1-373.g4e45e6b4.el9.x86_64 4/37 2024-09-21T09:40:40.573 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-osd-2:19.1.1-373.g4e45e6b4.el9.x86_64 5/37 2024-09-21T09:40:40.573 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-mgr-modules-core-2:19.1.1-373.g4e45e6b4.el9.n 6/37 2024-09-21T09:40:40.573 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/37 2024-09-21T09:40:40.573 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/37 2024-09-21T09:40:40.573 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/37 2024-09-21T09:40:40.573 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/37 2024-09-21T09:40:40.573 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/37 2024-09-21T09:40:40.573 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/37 2024-09-21T09:40:40.574 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : lua-5.4.4-4.el9.x86_64 13/37 2024-09-21T09:40:40.574 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 14/37 2024-09-21T09:40:40.574 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 15/37 2024-09-21T09:40:40.574 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 16/37 2024-09-21T09:40:40.574 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 17/37 2024-09-21T09:40:40.574 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 18/37 2024-09-21T09:40:40.574 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 19/37 2024-09-21T09:40:40.574 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 20/37 2024-09-21T09:40:40.574 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 21/37 2024-09-21T09:40:40.574 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 22/37 2024-09-21T09:40:40.574 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 23/37 2024-09-21T09:40:40.574 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 24/37 2024-09-21T09:40:40.574 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 25/37 2024-09-21T09:40:40.575 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 26/37 2024-09-21T09:40:40.575 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 27/37 2024-09-21T09:40:40.576 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 28/37 2024-09-21T09:40:40.576 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 29/37 2024-09-21T09:40:40.576 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 30/37 2024-09-21T09:40:40.576 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 31/37 2024-09-21T09:40:40.576 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 32/37 2024-09-21T09:40:40.576 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 33/37 2024-09-21T09:40:40.576 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 34/37 2024-09-21T09:40:40.576 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 35/37 2024-09-21T09:40:40.576 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 36/37 2024-09-21T09:40:41.519 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 37/37 2024-09-21T09:40:41.519 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:41.519 INFO:teuthology.orchestra.run.smithi092.stdout:Installed: 2024-09-21T09:40:41.519 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:41.520 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mds-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:41.520 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mgr-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:41.520 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mgr-modules-core-2:19.1.1-373.g4e45e6b4.el9.noarch 2024-09-21T09:40:41.520 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mon-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:41.520 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-osd-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:41.520 INFO:teuthology.orchestra.run.smithi092.stdout: lua-5.4.4-4.el9.x86_64 2024-09-21T09:40:41.520 INFO:teuthology.orchestra.run.smithi092.stdout: lua-devel-5.4.4-4.el9.x86_64 2024-09-21T09:40:41.520 INFO:teuthology.orchestra.run.smithi092.stdout: luarocks-3.9.2-5.el9.noarch 2024-09-21T09:40:41.520 INFO:teuthology.orchestra.run.smithi092.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-21T09:40:41.520 INFO:teuthology.orchestra.run.smithi092.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-21T09:40:41.520 INFO:teuthology.orchestra.run.smithi092.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-21T09:40:41.520 INFO:teuthology.orchestra.run.smithi092.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-21T09:40:41.520 INFO:teuthology.orchestra.run.smithi092.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-21T09:40:41.520 INFO:teuthology.orchestra.run.smithi092.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-21T09:40:41.521 INFO:teuthology.orchestra.run.smithi092.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-21T09:40:41.521 INFO:teuthology.orchestra.run.smithi092.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-21T09:40:41.521 INFO:teuthology.orchestra.run.smithi092.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-21T09:40:41.521 INFO:teuthology.orchestra.run.smithi092.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-21T09:40:41.521 INFO:teuthology.orchestra.run.smithi092.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-21T09:40:41.521 INFO:teuthology.orchestra.run.smithi092.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-21T09:40:41.521 INFO:teuthology.orchestra.run.smithi092.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-21T09:40:41.521 INFO:teuthology.orchestra.run.smithi092.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-21T09:40:41.521 INFO:teuthology.orchestra.run.smithi092.stdout: python3-packaging-20.9-5.el9.noarch 2024-09-21T09:40:41.521 INFO:teuthology.orchestra.run.smithi092.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-21T09:40:41.521 INFO:teuthology.orchestra.run.smithi092.stdout: python3-ply-3.11-14.el9.noarch 2024-09-21T09:40:41.521 INFO:teuthology.orchestra.run.smithi092.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-21T09:40:41.521 INFO:teuthology.orchestra.run.smithi092.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-21T09:40:41.522 INFO:teuthology.orchestra.run.smithi092.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-21T09:40:41.522 INFO:teuthology.orchestra.run.smithi092.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-21T09:40:41.522 INFO:teuthology.orchestra.run.smithi092.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-21T09:40:41.522 INFO:teuthology.orchestra.run.smithi092.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-21T09:40:41.522 INFO:teuthology.orchestra.run.smithi092.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-21T09:40:41.522 INFO:teuthology.orchestra.run.smithi092.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-21T09:40:41.522 INFO:teuthology.orchestra.run.smithi092.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-21T09:40:41.522 INFO:teuthology.orchestra.run.smithi092.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-21T09:40:41.522 INFO:teuthology.orchestra.run.smithi092.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-21T09:40:41.522 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:41.522 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! 2024-09-21T09:40:41.839 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum -y install ceph-base 2024-09-21T09:40:42.429 INFO:teuthology.orchestra.run.smithi092.stdout:Last metadata expiration check: 0:00:53 ago on Sat 21 Sep 2024 09:39:49 AM UTC. 2024-09-21T09:40:42.517 INFO:teuthology.orchestra.run.smithi092.stdout:Package ceph-base-2:19.1.1-373.g4e45e6b4.el9.x86_64 is already installed. 2024-09-21T09:40:42.559 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. 2024-09-21T09:40:42.560 INFO:teuthology.orchestra.run.smithi092.stdout:Nothing to do. 2024-09-21T09:40:42.560 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! 2024-09-21T09:40:42.621 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum -y install ceph-immutable-object-cache 2024-09-21T09:40:43.202 INFO:teuthology.orchestra.run.smithi092.stdout:Last metadata expiration check: 0:00:54 ago on Sat 21 Sep 2024 09:39:49 AM UTC. 2024-09-21T09:40:43.334 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. 2024-09-21T09:40:43.335 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:40:43.335 INFO:teuthology.orchestra.run.smithi092.stdout: Package Arch Version Repo Size 2024-09-21T09:40:43.335 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:40:43.335 INFO:teuthology.orchestra.run.smithi092.stdout:Installing: 2024-09-21T09:40:43.336 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-immutable-object-cache x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 145 k 2024-09-21T09:40:43.336 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:43.336 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction Summary 2024-09-21T09:40:43.336 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:40:43.336 INFO:teuthology.orchestra.run.smithi092.stdout:Install 1 Package 2024-09-21T09:40:43.336 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:43.336 INFO:teuthology.orchestra.run.smithi092.stdout:Total download size: 145 k 2024-09-21T09:40:43.336 INFO:teuthology.orchestra.run.smithi092.stdout:Installed size: 435 k 2024-09-21T09:40:43.336 INFO:teuthology.orchestra.run.smithi092.stdout:Downloading Packages: 2024-09-21T09:40:43.559 INFO:teuthology.orchestra.run.smithi092.stdout:ceph-immutable-object-cache-19.1.1-373.g4e45e6b 652 kB/s | 145 kB 00:00 2024-09-21T09:40:43.559 INFO:teuthology.orchestra.run.smithi092.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:40:43.560 INFO:teuthology.orchestra.run.smithi092.stdout:Total 648 kB/s | 145 kB 00:00 2024-09-21T09:40:43.560 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction check 2024-09-21T09:40:43.567 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction check succeeded. 2024-09-21T09:40:43.568 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction test 2024-09-21T09:40:43.625 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction test succeeded. 2024-09-21T09:40:43.625 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction 2024-09-21T09:40:43.780 INFO:teuthology.orchestra.run.smithi092.stdout: Preparing : 1/1 2024-09-21T09:40:43.834 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : ceph-immutable-object-cache-2:19.1.1-373.g4e45e6b4.e 1/1 2024-09-21T09:40:43.859 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.1.1-373.g4e45e6b4.e 1/1 2024-09-21T09:40:43.859 INFO:teuthology.orchestra.run.smithi092.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:40:43.859 INFO:teuthology.orchestra.run.smithi092.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-21T09:40:43.859 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:44.780 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-immutable-object-cache-2:19.1.1-373.g4e45e6b4.e 1/1 2024-09-21T09:40:44.780 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:44.780 INFO:teuthology.orchestra.run.smithi092.stdout:Installed: 2024-09-21T09:40:44.780 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-immutable-object-cache-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:44.780 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:44.780 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! 2024-09-21T09:40:45.013 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum -y install ceph-mgr 2024-09-21T09:40:45.615 INFO:teuthology.orchestra.run.smithi092.stdout:Last metadata expiration check: 0:00:56 ago on Sat 21 Sep 2024 09:39:49 AM UTC. 2024-09-21T09:40:45.704 INFO:teuthology.orchestra.run.smithi092.stdout:Package ceph-mgr-2:19.1.1-373.g4e45e6b4.el9.x86_64 is already installed. 2024-09-21T09:40:45.746 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. 2024-09-21T09:40:45.747 INFO:teuthology.orchestra.run.smithi092.stdout:Nothing to do. 2024-09-21T09:40:45.747 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! 2024-09-21T09:40:45.811 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum -y install ceph-fuse 2024-09-21T09:40:46.386 INFO:teuthology.orchestra.run.smithi092.stdout:Last metadata expiration check: 0:00:57 ago on Sat 21 Sep 2024 09:39:49 AM UTC. 2024-09-21T09:40:46.517 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. 2024-09-21T09:40:46.518 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:40:46.518 INFO:teuthology.orchestra.run.smithi092.stdout: Package Arch Version Repository Size 2024-09-21T09:40:46.518 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:40:46.518 INFO:teuthology.orchestra.run.smithi092.stdout:Installing: 2024-09-21T09:40:46.518 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-fuse x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 897 k 2024-09-21T09:40:46.518 INFO:teuthology.orchestra.run.smithi092.stdout:Installing dependencies: 2024-09-21T09:40:46.518 INFO:teuthology.orchestra.run.smithi092.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-21T09:40:46.518 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:46.518 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction Summary 2024-09-21T09:40:46.518 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:40:46.518 INFO:teuthology.orchestra.run.smithi092.stdout:Install 2 Packages 2024-09-21T09:40:46.518 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:46.519 INFO:teuthology.orchestra.run.smithi092.stdout:Total download size: 976 k 2024-09-21T09:40:46.519 INFO:teuthology.orchestra.run.smithi092.stdout:Installed size: 2.9 M 2024-09-21T09:40:46.519 INFO:teuthology.orchestra.run.smithi092.stdout:Downloading Packages: 2024-09-21T09:40:47.055 INFO:teuthology.orchestra.run.smithi092.stdout:(1/2): ceph-fuse-19.1.1-373.g4e45e6b4.el9.x86_6 3.1 MB/s | 897 kB 00:00 2024-09-21T09:40:47.239 INFO:teuthology.orchestra.run.smithi092.stdout:(2/2): fuse-2.9.9-16.el9.x86_64.rpm 172 kB/s | 80 kB 00:00 2024-09-21T09:40:47.239 INFO:teuthology.orchestra.run.smithi092.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:40:47.240 INFO:teuthology.orchestra.run.smithi092.stdout:Total 1.3 MB/s | 976 kB 00:00 2024-09-21T09:40:47.251 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction check 2024-09-21T09:40:47.273 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction check succeeded. 2024-09-21T09:40:47.273 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction test 2024-09-21T09:40:47.351 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction test succeeded. 2024-09-21T09:40:47.352 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction 2024-09-21T09:40:47.536 INFO:teuthology.orchestra.run.smithi092.stdout: Preparing : 1/1 2024-09-21T09:40:47.674 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-21T09:40:47.736 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : ceph-fuse-2:19.1.1-373.g4e45e6b4.el9.x86_64 2/2 2024-09-21T09:40:48.464 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-fuse-2:19.1.1-373.g4e45e6b4.el9.x86_64 2/2 2024-09-21T09:40:48.464 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-fuse-2:19.1.1-373.g4e45e6b4.el9.x86_64 1/2 2024-09-21T09:40:48.722 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-21T09:40:48.723 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:48.723 INFO:teuthology.orchestra.run.smithi092.stdout:Installed: 2024-09-21T09:40:48.723 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-fuse-2:19.1.1-373.g4e45e6b4.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-21T09:40:48.723 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:48.723 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! 2024-09-21T09:40:49.002 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum -y install librados-devel 2024-09-21T09:40:49.595 INFO:teuthology.orchestra.run.smithi092.stdout:Last metadata expiration check: 0:01:00 ago on Sat 21 Sep 2024 09:39:49 AM UTC. 2024-09-21T09:40:49.729 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. 2024-09-21T09:40:49.729 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:40:49.730 INFO:teuthology.orchestra.run.smithi092.stdout: Package Arch Version Repo Size 2024-09-21T09:40:49.730 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:40:49.730 INFO:teuthology.orchestra.run.smithi092.stdout:Installing: 2024-09-21T09:40:49.730 INFO:teuthology.orchestra.run.smithi092.stdout: librados-devel x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 127 k 2024-09-21T09:40:49.730 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:49.730 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction Summary 2024-09-21T09:40:49.730 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:40:49.730 INFO:teuthology.orchestra.run.smithi092.stdout:Install 1 Package 2024-09-21T09:40:49.730 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:49.730 INFO:teuthology.orchestra.run.smithi092.stdout:Total download size: 127 k 2024-09-21T09:40:49.730 INFO:teuthology.orchestra.run.smithi092.stdout:Installed size: 456 k 2024-09-21T09:40:49.730 INFO:teuthology.orchestra.run.smithi092.stdout:Downloading Packages: 2024-09-21T09:40:49.926 INFO:teuthology.orchestra.run.smithi092.stdout:librados-devel-19.1.1-373.g4e45e6b4.el9.x86_64. 649 kB/s | 127 kB 00:00 2024-09-21T09:40:49.926 INFO:teuthology.orchestra.run.smithi092.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:40:49.927 INFO:teuthology.orchestra.run.smithi092.stdout:Total 644 kB/s | 127 kB 00:00 2024-09-21T09:40:49.927 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction check 2024-09-21T09:40:49.932 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction check succeeded. 2024-09-21T09:40:49.932 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction test 2024-09-21T09:40:49.990 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction test succeeded. 2024-09-21T09:40:49.990 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction 2024-09-21T09:40:50.128 INFO:teuthology.orchestra.run.smithi092.stdout: Preparing : 1/1 2024-09-21T09:40:50.229 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : librados-devel-2:19.1.1-373.g4e45e6b4.el9.x86_64 1/1 2024-09-21T09:40:50.627 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: librados-devel-2:19.1.1-373.g4e45e6b4.el9.x86_64 1/1 2024-09-21T09:40:50.874 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : librados-devel-2:19.1.1-373.g4e45e6b4.el9.x86_64 1/1 2024-09-21T09:40:50.874 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:50.874 INFO:teuthology.orchestra.run.smithi092.stdout:Installed: 2024-09-21T09:40:50.875 INFO:teuthology.orchestra.run.smithi092.stdout: librados-devel-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:50.875 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:50.875 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! 2024-09-21T09:40:51.128 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum -y install libcephfs2 2024-09-21T09:40:51.707 INFO:teuthology.orchestra.run.smithi092.stdout:Last metadata expiration check: 0:01:02 ago on Sat 21 Sep 2024 09:39:49 AM UTC. 2024-09-21T09:40:51.796 INFO:teuthology.orchestra.run.smithi092.stdout:Package libcephfs2-2:19.1.1-373.g4e45e6b4.el9.x86_64 is already installed. 2024-09-21T09:40:51.840 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. 2024-09-21T09:40:51.841 INFO:teuthology.orchestra.run.smithi092.stdout:Nothing to do. 2024-09-21T09:40:51.842 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! 2024-09-21T09:40:51.904 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum -y install libcephfs-devel 2024-09-21T09:40:52.477 INFO:teuthology.orchestra.run.smithi092.stdout:Last metadata expiration check: 0:01:03 ago on Sat 21 Sep 2024 09:39:49 AM UTC. 2024-09-21T09:40:52.608 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. 2024-09-21T09:40:52.608 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:40:52.608 INFO:teuthology.orchestra.run.smithi092.stdout: Package Arch Version Repo Size 2024-09-21T09:40:52.608 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:40:52.609 INFO:teuthology.orchestra.run.smithi092.stdout:Installing: 2024-09-21T09:40:52.609 INFO:teuthology.orchestra.run.smithi092.stdout: libcephfs-devel x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 32 k 2024-09-21T09:40:52.609 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:52.609 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction Summary 2024-09-21T09:40:52.609 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:40:52.609 INFO:teuthology.orchestra.run.smithi092.stdout:Install 1 Package 2024-09-21T09:40:52.609 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:52.609 INFO:teuthology.orchestra.run.smithi092.stdout:Total download size: 32 k 2024-09-21T09:40:52.609 INFO:teuthology.orchestra.run.smithi092.stdout:Installed size: 142 k 2024-09-21T09:40:52.609 INFO:teuthology.orchestra.run.smithi092.stdout:Downloading Packages: 2024-09-21T09:40:52.761 INFO:teuthology.orchestra.run.smithi092.stdout:libcephfs-devel-19.1.1-373.g4e45e6b4.el9.x86_64 208 kB/s | 32 kB 00:00 2024-09-21T09:40:52.762 INFO:teuthology.orchestra.run.smithi092.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:40:52.762 INFO:teuthology.orchestra.run.smithi092.stdout:Total 207 kB/s | 32 kB 00:00 2024-09-21T09:40:52.762 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction check 2024-09-21T09:40:52.765 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction check succeeded. 2024-09-21T09:40:52.766 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction test 2024-09-21T09:40:52.785 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction test succeeded. 2024-09-21T09:40:52.786 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction 2024-09-21T09:40:52.864 INFO:teuthology.orchestra.run.smithi092.stdout: Preparing : 1/1 2024-09-21T09:40:52.973 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : libcephfs-devel-2:19.1.1-373.g4e45e6b4.el9.x86_64 1/1 2024-09-21T09:40:53.309 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: libcephfs-devel-2:19.1.1-373.g4e45e6b4.el9.x86_64 1/1 2024-09-21T09:40:53.531 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : libcephfs-devel-2:19.1.1-373.g4e45e6b4.el9.x86_64 1/1 2024-09-21T09:40:53.531 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:53.531 INFO:teuthology.orchestra.run.smithi092.stdout:Installed: 2024-09-21T09:40:53.531 INFO:teuthology.orchestra.run.smithi092.stdout: libcephfs-devel-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:40:53.531 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:53.531 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! 2024-09-21T09:40:53.702 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum -y install librados2 2024-09-21T09:40:54.279 INFO:teuthology.orchestra.run.smithi092.stdout:Last metadata expiration check: 0:01:05 ago on Sat 21 Sep 2024 09:39:49 AM UTC. 2024-09-21T09:40:54.369 INFO:teuthology.orchestra.run.smithi092.stdout:Package librados2-2:19.1.1-373.g4e45e6b4.el9.x86_64 is already installed. 2024-09-21T09:40:54.410 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. 2024-09-21T09:40:54.412 INFO:teuthology.orchestra.run.smithi092.stdout:Nothing to do. 2024-09-21T09:40:54.412 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! 2024-09-21T09:40:54.475 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum -y install librbd1 2024-09-21T09:40:55.049 INFO:teuthology.orchestra.run.smithi092.stdout:Last metadata expiration check: 0:01:06 ago on Sat 21 Sep 2024 09:39:49 AM UTC. 2024-09-21T09:40:55.136 INFO:teuthology.orchestra.run.smithi092.stdout:Package librbd1-2:19.1.1-373.g4e45e6b4.el9.x86_64 is already installed. 2024-09-21T09:40:55.177 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. 2024-09-21T09:40:55.179 INFO:teuthology.orchestra.run.smithi092.stdout:Nothing to do. 2024-09-21T09:40:55.179 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! 2024-09-21T09:40:55.240 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum -y install python3-rados 2024-09-21T09:40:55.820 INFO:teuthology.orchestra.run.smithi092.stdout:Last metadata expiration check: 0:01:06 ago on Sat 21 Sep 2024 09:39:49 AM UTC. 2024-09-21T09:40:55.908 INFO:teuthology.orchestra.run.smithi092.stdout:Package python3-rados-2:19.1.1-373.g4e45e6b4.el9.x86_64 is already installed. 2024-09-21T09:40:55.950 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. 2024-09-21T09:40:55.951 INFO:teuthology.orchestra.run.smithi092.stdout:Nothing to do. 2024-09-21T09:40:55.951 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! 2024-09-21T09:40:56.017 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum -y install python3-rgw 2024-09-21T09:40:56.595 INFO:teuthology.orchestra.run.smithi092.stdout:Last metadata expiration check: 0:01:07 ago on Sat 21 Sep 2024 09:39:49 AM UTC. 2024-09-21T09:40:56.684 INFO:teuthology.orchestra.run.smithi092.stdout:Package python3-rgw-2:19.1.1-373.g4e45e6b4.el9.x86_64 is already installed. 2024-09-21T09:40:56.726 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. 2024-09-21T09:40:56.727 INFO:teuthology.orchestra.run.smithi092.stdout:Nothing to do. 2024-09-21T09:40:56.727 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! 2024-09-21T09:40:56.790 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum -y install python3-cephfs 2024-09-21T09:40:57.365 INFO:teuthology.orchestra.run.smithi092.stdout:Last metadata expiration check: 0:01:08 ago on Sat 21 Sep 2024 09:39:49 AM UTC. 2024-09-21T09:40:57.454 INFO:teuthology.orchestra.run.smithi092.stdout:Package python3-cephfs-2:19.1.1-373.g4e45e6b4.el9.x86_64 is already installed. 2024-09-21T09:40:57.496 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. 2024-09-21T09:40:57.497 INFO:teuthology.orchestra.run.smithi092.stdout:Nothing to do. 2024-09-21T09:40:57.497 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! 2024-09-21T09:40:57.557 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum -y install python3-rbd 2024-09-21T09:40:58.140 INFO:teuthology.orchestra.run.smithi092.stdout:Last metadata expiration check: 0:01:09 ago on Sat 21 Sep 2024 09:39:49 AM UTC. 2024-09-21T09:40:58.232 INFO:teuthology.orchestra.run.smithi092.stdout:Package python3-rbd-2:19.1.1-373.g4e45e6b4.el9.x86_64 is already installed. 2024-09-21T09:40:58.276 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. 2024-09-21T09:40:58.277 INFO:teuthology.orchestra.run.smithi092.stdout:Nothing to do. 2024-09-21T09:40:58.277 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! 2024-09-21T09:40:58.341 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum -y install rbd-fuse 2024-09-21T09:40:58.948 INFO:teuthology.orchestra.run.smithi092.stdout:Last metadata expiration check: 0:01:09 ago on Sat 21 Sep 2024 09:39:49 AM UTC. 2024-09-21T09:40:59.084 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. 2024-09-21T09:40:59.085 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:40:59.085 INFO:teuthology.orchestra.run.smithi092.stdout: Package Architecture Version Repository Size 2024-09-21T09:40:59.085 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:40:59.086 INFO:teuthology.orchestra.run.smithi092.stdout:Installing: 2024-09-21T09:40:59.086 INFO:teuthology.orchestra.run.smithi092.stdout: rbd-fuse x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 85 k 2024-09-21T09:40:59.086 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:59.086 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction Summary 2024-09-21T09:40:59.086 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:40:59.086 INFO:teuthology.orchestra.run.smithi092.stdout:Install 1 Package 2024-09-21T09:40:59.086 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:40:59.086 INFO:teuthology.orchestra.run.smithi092.stdout:Total download size: 85 k 2024-09-21T09:40:59.086 INFO:teuthology.orchestra.run.smithi092.stdout:Installed size: 231 k 2024-09-21T09:40:59.086 INFO:teuthology.orchestra.run.smithi092.stdout:Downloading Packages: 2024-09-21T09:40:59.281 INFO:teuthology.orchestra.run.smithi092.stdout:rbd-fuse-19.1.1-373.g4e45e6b4.el9.x86_64.rpm 438 kB/s | 85 kB 00:00 2024-09-21T09:40:59.282 INFO:teuthology.orchestra.run.smithi092.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:40:59.282 INFO:teuthology.orchestra.run.smithi092.stdout:Total 434 kB/s | 85 kB 00:00 2024-09-21T09:40:59.283 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction check 2024-09-21T09:40:59.290 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction check succeeded. 2024-09-21T09:40:59.290 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction test 2024-09-21T09:40:59.347 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction test succeeded. 2024-09-21T09:40:59.347 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction 2024-09-21T09:40:59.496 INFO:teuthology.orchestra.run.smithi092.stdout: Preparing : 1/1 2024-09-21T09:40:59.601 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : rbd-fuse-2:19.1.1-373.g4e45e6b4.el9.x86_64 1/1 2024-09-21T09:40:59.982 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: rbd-fuse-2:19.1.1-373.g4e45e6b4.el9.x86_64 1/1 2024-09-21T09:41:00.228 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : rbd-fuse-2:19.1.1-373.g4e45e6b4.el9.x86_64 1/1 2024-09-21T09:41:00.228 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:00.228 INFO:teuthology.orchestra.run.smithi092.stdout:Installed: 2024-09-21T09:41:00.228 INFO:teuthology.orchestra.run.smithi092.stdout: rbd-fuse-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:41:00.228 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:00.228 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! 2024-09-21T09:41:00.452 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum -y install rbd-mirror 2024-09-21T09:41:01.045 INFO:teuthology.orchestra.run.smithi092.stdout:Last metadata expiration check: 0:01:12 ago on Sat 21 Sep 2024 09:39:49 AM UTC. 2024-09-21T09:41:01.181 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. 2024-09-21T09:41:01.182 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:41:01.182 INFO:teuthology.orchestra.run.smithi092.stdout: Package Arch Version Repo Size 2024-09-21T09:41:01.182 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:41:01.182 INFO:teuthology.orchestra.run.smithi092.stdout:Installing: 2024-09-21T09:41:01.182 INFO:teuthology.orchestra.run.smithi092.stdout: rbd-mirror x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 3.1 M 2024-09-21T09:41:01.182 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:01.182 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction Summary 2024-09-21T09:41:01.182 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:41:01.183 INFO:teuthology.orchestra.run.smithi092.stdout:Install 1 Package 2024-09-21T09:41:01.183 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:01.183 INFO:teuthology.orchestra.run.smithi092.stdout:Total download size: 3.1 M 2024-09-21T09:41:01.183 INFO:teuthology.orchestra.run.smithi092.stdout:Installed size: 13 M 2024-09-21T09:41:01.183 INFO:teuthology.orchestra.run.smithi092.stdout:Downloading Packages: 2024-09-21T09:41:01.518 INFO:teuthology.orchestra.run.smithi092.stdout:rbd-mirror-19.1.1-373.g4e45e6b4.el9.x86_64.rpm 9.3 MB/s | 3.1 MB 00:00 2024-09-21T09:41:01.518 INFO:teuthology.orchestra.run.smithi092.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:41:01.518 INFO:teuthology.orchestra.run.smithi092.stdout:Total 9.2 MB/s | 3.1 MB 00:00 2024-09-21T09:41:01.519 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction check 2024-09-21T09:41:01.527 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction check succeeded. 2024-09-21T09:41:01.527 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction test 2024-09-21T09:41:01.601 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction test succeeded. 2024-09-21T09:41:01.601 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction 2024-09-21T09:41:02.021 INFO:teuthology.orchestra.run.smithi092.stdout: Preparing : 1/1 2024-09-21T09:41:02.077 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : rbd-mirror-2:19.1.1-373.g4e45e6b4.el9.x86_64 1/1 2024-09-21T09:41:02.102 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: rbd-mirror-2:19.1.1-373.g4e45e6b4.el9.x86_64 1/1 2024-09-21T09:41:02.102 INFO:teuthology.orchestra.run.smithi092.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:41:02.102 INFO:teuthology.orchestra.run.smithi092.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-21T09:41:02.102 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:41:02.102 INFO:teuthology.orchestra.run.smithi092.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:41:02.102 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:03.131 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : rbd-mirror-2:19.1.1-373.g4e45e6b4.el9.x86_64 1/1 2024-09-21T09:41:03.131 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:03.131 INFO:teuthology.orchestra.run.smithi092.stdout:Installed: 2024-09-21T09:41:03.131 INFO:teuthology.orchestra.run.smithi092.stdout: rbd-mirror-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:41:03.131 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:03.131 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! 2024-09-21T09:41:03.371 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum -y install rbd-nbd 2024-09-21T09:41:03.965 INFO:teuthology.orchestra.run.smithi092.stdout:Last metadata expiration check: 0:01:14 ago on Sat 21 Sep 2024 09:39:49 AM UTC. 2024-09-21T09:41:04.098 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. 2024-09-21T09:41:04.099 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:41:04.099 INFO:teuthology.orchestra.run.smithi092.stdout: Package Architecture Version Repository Size 2024-09-21T09:41:04.099 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:41:04.099 INFO:teuthology.orchestra.run.smithi092.stdout:Installing: 2024-09-21T09:41:04.099 INFO:teuthology.orchestra.run.smithi092.stdout: rbd-nbd x86_64 2:19.1.1-373.g4e45e6b4.el9 ceph 170 k 2024-09-21T09:41:04.099 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:04.099 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction Summary 2024-09-21T09:41:04.099 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:41:04.099 INFO:teuthology.orchestra.run.smithi092.stdout:Install 1 Package 2024-09-21T09:41:04.099 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:04.100 INFO:teuthology.orchestra.run.smithi092.stdout:Total download size: 170 k 2024-09-21T09:41:04.100 INFO:teuthology.orchestra.run.smithi092.stdout:Installed size: 494 k 2024-09-21T09:41:04.100 INFO:teuthology.orchestra.run.smithi092.stdout:Downloading Packages: 2024-09-21T09:41:04.321 INFO:teuthology.orchestra.run.smithi092.stdout:rbd-nbd-19.1.1-373.g4e45e6b4.el9.x86_64.rpm 773 kB/s | 170 kB 00:00 2024-09-21T09:41:04.321 INFO:teuthology.orchestra.run.smithi092.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:41:04.322 INFO:teuthology.orchestra.run.smithi092.stdout:Total 767 kB/s | 170 kB 00:00 2024-09-21T09:41:04.322 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction check 2024-09-21T09:41:04.329 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction check succeeded. 2024-09-21T09:41:04.330 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction test 2024-09-21T09:41:04.387 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction test succeeded. 2024-09-21T09:41:04.387 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction 2024-09-21T09:41:04.538 INFO:teuthology.orchestra.run.smithi092.stdout: Preparing : 1/1 2024-09-21T09:41:04.633 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : rbd-nbd-2:19.1.1-373.g4e45e6b4.el9.x86_64 1/1 2024-09-21T09:41:05.061 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: rbd-nbd-2:19.1.1-373.g4e45e6b4.el9.x86_64 1/1 2024-09-21T09:41:05.296 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : rbd-nbd-2:19.1.1-373.g4e45e6b4.el9.x86_64 1/1 2024-09-21T09:41:05.296 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:05.297 INFO:teuthology.orchestra.run.smithi092.stdout:Installed: 2024-09-21T09:41:05.297 INFO:teuthology.orchestra.run.smithi092.stdout: rbd-nbd-2:19.1.1-373.g4e45e6b4.el9.x86_64 2024-09-21T09:41:05.297 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:05.297 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! 2024-09-21T09:41:05.537 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum -y install librados2 2024-09-21T09:41:06.129 INFO:teuthology.orchestra.run.smithi092.stdout:Last metadata expiration check: 0:01:17 ago on Sat 21 Sep 2024 09:39:49 AM UTC. 2024-09-21T09:41:06.220 INFO:teuthology.orchestra.run.smithi092.stdout:Package librados2-2:19.1.1-373.g4e45e6b4.el9.x86_64 is already installed. 2024-09-21T09:41:06.263 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. 2024-09-21T09:41:06.264 INFO:teuthology.orchestra.run.smithi092.stdout:Nothing to do. 2024-09-21T09:41:06.264 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! 2024-09-21T09:41:06.325 DEBUG:teuthology.parallel:result is None 2024-09-21T09:41:06.325 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:41:06.325 INFO:teuthology.packaging:ref: None 2024-09-21T09:41:06.325 INFO:teuthology.packaging:tag: None 2024-09-21T09:41:06.325 INFO:teuthology.packaging:branch: squid 2024-09-21T09:41:06.325 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:41:06.325 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-09-21T09:41:06.562 DEBUG:teuthology.orchestra.run.smithi092:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-21T09:41:06.584 INFO:teuthology.orchestra.run.smithi092.stdout:19.1.1-373.g4e45e6b4.el9 2024-09-21T09:41:06.585 INFO:teuthology.packaging:The installed version of ceph is 19.1.1-373.g4e45e6b4.el9 2024-09-21T09:41:06.585 INFO:teuthology.task.install:The correct ceph version 19.1.1-373.g4e45e6b4 is installed. 2024-09-21T09:41:06.586 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-21T09:41:06.586 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:41:06.586 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-21T09:41:06.649 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-21T09:41:06.650 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:41:06.650 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd of=/usr/bin/daemon-helper 2024-09-21T09:41:06.714 DEBUG:teuthology.orchestra.run.smithi092:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-21T09:41:06.778 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-21T09:41:06.778 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:41:06.778 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-21T09:41:06.841 DEBUG:teuthology.orchestra.run.smithi092:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-21T09:41:06.906 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-21T09:41:06.906 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:41:06.906 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd of=/usr/bin/stdin-killer 2024-09-21T09:41:06.971 DEBUG:teuthology.orchestra.run.smithi092:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-21T09:41:07.036 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:41:07.052 INFO:teuthology.task.print:**** done installing squid 2024-09-21T09:41:07.052 INFO:teuthology.run_tasks:Running task ceph... 2024-09-21T09:41:07.189 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2024-09-21T09:41:07.189 DEBUG:teuthology.orchestra.run.smithi092:> sudo chmod 777 /var/log/ceph 2024-09-21T09:41:07.213 INFO:tasks.ceph:Disabling ceph logrotate... 2024-09-21T09:41:07.214 DEBUG:teuthology.orchestra.run.smithi092:> sudo rm -f -- /etc/logrotate.d/ceph 2024-09-21T09:41:07.278 INFO:tasks.ceph:Creating extra log directories... 2024-09-21T09:41:07.278 DEBUG:teuthology.orchestra.run.smithi092:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2024-09-21T09:41:07.350 INFO:tasks.ceph:Creating ceph cluster ceph... 2024-09-21T09:41:07.350 INFO:tasks.ceph:config {'conf': {'global': {'mon warn on pool no app': False, 'ms bind msgr2': False, 'bluestore warn on legacy statfs': False, 'bluestore warn on no per pool omap': False, 'mon pg warn min per osd': 0}, 'mgr': {'client mount timeout': 30, 'debug client': 20, 'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon down mkfs grace': 300, 'mon warn on osd down out interval zero': False}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['overall HEALTH_', '\\(FS_', '\\(MDS_', '\\(OSD_', '\\(MON_DOWN\\)', '\\(CACHE_POOL_', '\\(POOL_', '\\(MGR_DOWN\\)', '\\(PG_', '\\(SMALLER_PGP_NUM\\)', 'Monitor daemon marked osd', 'Behind on trimming', 'Manager daemon', '\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'fs.*is degraded', 'filesystem is degraded', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'filesystem is offline', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'insufficient standby MDS daemons available', 'MDS_UP_LESS_THAN_MAX', 'online, but wants', 'filesystem is online with fewer MDS than max_mds', 'POOL_APP_NOT_ENABLED', 'do not have an application enabled', 'overall HEALTH_', 'Replacing daemon', 'deprecated feature inline_data', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding', 'PG_AVAILABILITY', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'scrub mismatch', 'ScrubResult', 'wrongly marked', '\\(POOL_APP_NOT_ENABLED\\)', '\\(SLOW_OPS\\)', 'overall HEALTH_', '\\(MON_MSGR2_NOT_ENABLED\\)', 'slow request'], 'cpu_profile': set(), 'cluster': 'ceph', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2024-09-21T09:41:07.350 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913659', 'branch': 'main', 'description': 'upgrade/cephfs/nofs/{bluestore-bitmap centos_9.stream conf/{client mds mgr mon osd} no-mds-cluster overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn pg_health} tasks/{0-from/squid 1-upgrade}}', 'email': 'ceph-qa@ceph.com', 'first_in_suite': False, 'job_id': '7913659', 'kernel': {'kdb': 1, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'smithi', 'meta': [{'desc': 'install ceph/squid latest\n'}], 'name': 'teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 4, 'size': 10}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'main'}, 'ceph': {'cluster-conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 2, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mon': {'mon op complaint time': 120, 'mon warn on pool no app': False}, 'osd': {'osd op complaint time': 180}}, 'conf': {'global': {'bluestore warn on legacy statfs': False, 'bluestore warn on no per pool omap': False, 'mon pg warn min per osd': 0}, 'mgr': {'client mount timeout': 30, 'debug client': 20, 'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon down mkfs grace': 300, 'mon warn on osd down out interval zero': False}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'fs.*is degraded', 'filesystem is degraded', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'filesystem is offline', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'insufficient standby MDS daemons available', 'MDS_UP_LESS_THAN_MAX', 'online, but wants', 'filesystem is online with fewer MDS than max_mds', 'POOL_APP_NOT_ENABLED', 'do not have an application enabled', 'overall HEALTH_', 'Replacing daemon', 'deprecated feature inline_data', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding', 'PG_AVAILABILITY', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'scrub mismatch', 'ScrubResult', 'wrongly marked', '\\(POOL_APP_NOT_ENABLED\\)', '\\(SLOW_OPS\\)', 'overall HEALTH_', '\\(MON_MSGR2_NOT_ENABLED\\)', 'slow request'], 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'main', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}}, 'owner': 'scheduled_teuthology@teuthology', 'priority': 850, 'repo': 'https://git.ceph.com/ceph.git', 'roles': [['mon.a', 'mon.b', 'mon.c', 'mgr.x', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3']], 'seed': 5357, 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208', 'sleep_before_teardown': 0, 'subset': '31/32', 'suite': 'upgrade', 'suite_branch': 'main', 'suite_path': '/home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://git.ceph.com/ceph.git', 'suite_sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208', 'targets': {'smithi092.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBADqCvZHDIzgj3Qo8lQoI5Nwj8MJWM6J29P8Qgrc7j6VhXOZ8AyH90DS7pd7rxbDEa89Dzqyg9hYQfqgF1mfgEw='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': 1, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'branch': 'squid', 'exclude_packages': ['librados3', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'cephadm', 'ceph-volume'], 'extra_packages': ['librados2'], 'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}}, {'print': '**** done installing squid'}, {'ceph': {'conf': {'global': {'mon warn on pool no app': False, 'ms bind msgr2': False, 'bluestore warn on legacy statfs': False, 'bluestore warn on no per pool omap': False, 'mon pg warn min per osd': 0}, 'mgr': {'client mount timeout': 30, 'debug client': 20, 'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon down mkfs grace': 300, 'mon warn on osd down out interval zero': False}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'log-ignorelist': ['overall HEALTH_', '\\(FS_', '\\(MDS_', '\\(OSD_', '\\(MON_DOWN\\)', '\\(CACHE_POOL_', '\\(POOL_', '\\(MGR_DOWN\\)', '\\(PG_', '\\(SMALLER_PGP_NUM\\)', 'Monitor daemon marked osd', 'Behind on trimming', 'Manager daemon', '\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'fs.*is degraded', 'filesystem is degraded', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'filesystem is offline', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'insufficient standby MDS daemons available', 'MDS_UP_LESS_THAN_MAX', 'online, but wants', 'filesystem is online with fewer MDS than max_mds', 'POOL_APP_NOT_ENABLED', 'do not have an application enabled', 'overall HEALTH_', 'Replacing daemon', 'deprecated feature inline_data', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding', 'PG_AVAILABILITY', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'scrub mismatch', 'ScrubResult', 'wrongly marked', '\\(POOL_APP_NOT_ENABLED\\)', '\\(SLOW_OPS\\)', 'overall HEALTH_', '\\(MON_MSGR2_NOT_ENABLED\\)', 'slow request'], 'cluster-conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 2, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mon': {'mon op complaint time': 120, 'mon warn on pool no app': False}, 'osd': {'osd op complaint time': 180}}, 'flavor': 'default', 'fs': 'xfs', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208', 'cluster': 'ceph'}}, {'exec': {'osd.0': ['ceph osd set-require-min-compat-client squid']}}, {'print': '**** done ceph'}, {'print': '*** upgrading, no cephfs present'}, {'exec': {'mon.a': ['ceph fs dump']}}, {'install.upgrade': {'mon.a': None}}, {'print': '**** done install.upgrade'}, {'ceph.restart': {'daemons': ['mon.*', 'mgr.*'], 'mon-health-to-clog': False, 'wait-for-healthy': False}}, {'ceph.healthy': None}, {'ceph.restart': {'daemons': ['osd.*'], 'wait-for-healthy': False, 'wait-for-osds-up': True}}, {'exec': {'mon.a': ['ceph versions', 'ceph osd dump -f json-pretty', 'ceph fs dump', 'ceph osd require-osd-release squid', 'for f in `ceph osd pool ls` ; do ceph osd pool set $f pg_autoscale_mode off ; done']}}, {'ceph.healthy': None}, {'print': '**** done ceph.restart'}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': '3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a', 'timestamp': '2024-09-21_03:08:26', 'tube': 'smithi', 'user': 'teuthology', 'verbose': False, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.2902'} 2024-09-21T09:41:07.351 DEBUG:teuthology.orchestra.run.smithi092:> install -d -m0755 -- /home/ubuntu/cephtest/ceph.data 2024-09-21T09:41:07.398 DEBUG:teuthology.orchestra.run.smithi092:> sudo install -d -m0777 -- /var/run/ceph 2024-09-21T09:41:07.463 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:41:07.463 DEBUG:teuthology.orchestra.run.smithi092:> dd if=/scratch_devs of=/dev/stdout 2024-09-21T09:41:07.518 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-21T09:41:07.518 DEBUG:teuthology.orchestra.run.smithi092:> stat /dev/vg_nvme/lv_1 2024-09-21T09:41:07.572 INFO:teuthology.orchestra.run.smithi092.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-21T09:41:07.572 INFO:teuthology.orchestra.run.smithi092.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:41:07.572 INFO:teuthology.orchestra.run.smithi092.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-09-21T09:41:07.572 INFO:teuthology.orchestra.run.smithi092.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:41:07.573 INFO:teuthology.orchestra.run.smithi092.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:41:07.573 INFO:teuthology.orchestra.run.smithi092.stdout:Access: 2024-09-21 09:41:02.346945157 +0000 2024-09-21T09:41:07.573 INFO:teuthology.orchestra.run.smithi092.stdout:Modify: 2024-09-21 09:39:11.677937179 +0000 2024-09-21T09:41:07.573 INFO:teuthology.orchestra.run.smithi092.stdout:Change: 2024-09-21 09:39:11.677937179 +0000 2024-09-21T09:41:07.573 INFO:teuthology.orchestra.run.smithi092.stdout: Birth: 2024-09-21 09:39:11.677937179 +0000 2024-09-21T09:41:07.573 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-21T09:41:07.636 INFO:teuthology.orchestra.run.smithi092.stderr:1+0 records in 2024-09-21T09:41:07.636 INFO:teuthology.orchestra.run.smithi092.stderr:1+0 records out 2024-09-21T09:41:07.636 INFO:teuthology.orchestra.run.smithi092.stderr:512 bytes copied, 0.000262012 s, 2.0 MB/s 2024-09-21T09:41:07.637 DEBUG:teuthology.orchestra.run.smithi092:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-21T09:41:07.693 DEBUG:teuthology.orchestra.run.smithi092:> stat /dev/vg_nvme/lv_2 2024-09-21T09:41:07.748 INFO:teuthology.orchestra.run.smithi092.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-21T09:41:07.748 INFO:teuthology.orchestra.run.smithi092.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:41:07.748 INFO:teuthology.orchestra.run.smithi092.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-09-21T09:41:07.749 INFO:teuthology.orchestra.run.smithi092.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:41:07.749 INFO:teuthology.orchestra.run.smithi092.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:41:07.749 INFO:teuthology.orchestra.run.smithi092.stdout:Access: 2024-09-21 09:41:02.347945128 +0000 2024-09-21T09:41:07.749 INFO:teuthology.orchestra.run.smithi092.stdout:Modify: 2024-09-21 09:39:12.016927174 +0000 2024-09-21T09:41:07.749 INFO:teuthology.orchestra.run.smithi092.stdout:Change: 2024-09-21 09:39:12.016927174 +0000 2024-09-21T09:41:07.749 INFO:teuthology.orchestra.run.smithi092.stdout: Birth: 2024-09-21 09:39:12.016927174 +0000 2024-09-21T09:41:07.749 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-21T09:41:07.811 INFO:teuthology.orchestra.run.smithi092.stderr:1+0 records in 2024-09-21T09:41:07.811 INFO:teuthology.orchestra.run.smithi092.stderr:1+0 records out 2024-09-21T09:41:07.811 INFO:teuthology.orchestra.run.smithi092.stderr:512 bytes copied, 0.000181455 s, 2.8 MB/s 2024-09-21T09:41:07.813 DEBUG:teuthology.orchestra.run.smithi092:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-21T09:41:07.869 DEBUG:teuthology.orchestra.run.smithi092:> stat /dev/vg_nvme/lv_3 2024-09-21T09:41:07.925 INFO:teuthology.orchestra.run.smithi092.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-21T09:41:07.925 INFO:teuthology.orchestra.run.smithi092.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:41:07.925 INFO:teuthology.orchestra.run.smithi092.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-09-21T09:41:07.925 INFO:teuthology.orchestra.run.smithi092.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:41:07.925 INFO:teuthology.orchestra.run.smithi092.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:41:07.925 INFO:teuthology.orchestra.run.smithi092.stdout:Access: 2024-09-21 09:41:02.347945128 +0000 2024-09-21T09:41:07.925 INFO:teuthology.orchestra.run.smithi092.stdout:Modify: 2024-09-21 09:39:12.335917760 +0000 2024-09-21T09:41:07.925 INFO:teuthology.orchestra.run.smithi092.stdout:Change: 2024-09-21 09:39:12.335917760 +0000 2024-09-21T09:41:07.925 INFO:teuthology.orchestra.run.smithi092.stdout: Birth: 2024-09-21 09:39:12.335917760 +0000 2024-09-21T09:41:07.926 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-21T09:41:07.988 INFO:teuthology.orchestra.run.smithi092.stderr:1+0 records in 2024-09-21T09:41:07.989 INFO:teuthology.orchestra.run.smithi092.stderr:1+0 records out 2024-09-21T09:41:07.989 INFO:teuthology.orchestra.run.smithi092.stderr:512 bytes copied, 0.000212778 s, 2.4 MB/s 2024-09-21T09:41:07.990 DEBUG:teuthology.orchestra.run.smithi092:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-21T09:41:08.046 DEBUG:teuthology.orchestra.run.smithi092:> stat /dev/vg_nvme/lv_4 2024-09-21T09:41:08.101 INFO:teuthology.orchestra.run.smithi092.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-21T09:41:08.102 INFO:teuthology.orchestra.run.smithi092.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:41:08.102 INFO:teuthology.orchestra.run.smithi092.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-09-21T09:41:08.102 INFO:teuthology.orchestra.run.smithi092.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:41:08.102 INFO:teuthology.orchestra.run.smithi092.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:41:08.102 INFO:teuthology.orchestra.run.smithi092.stdout:Access: 2024-09-21 09:41:02.348945098 +0000 2024-09-21T09:41:08.102 INFO:teuthology.orchestra.run.smithi092.stdout:Modify: 2024-09-21 09:39:12.660908168 +0000 2024-09-21T09:41:08.102 INFO:teuthology.orchestra.run.smithi092.stdout:Change: 2024-09-21 09:39:12.660908168 +0000 2024-09-21T09:41:08.102 INFO:teuthology.orchestra.run.smithi092.stdout: Birth: 2024-09-21 09:39:12.660908168 +0000 2024-09-21T09:41:08.102 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-21T09:41:08.164 INFO:teuthology.orchestra.run.smithi092.stderr:1+0 records in 2024-09-21T09:41:08.164 INFO:teuthology.orchestra.run.smithi092.stderr:1+0 records out 2024-09-21T09:41:08.164 INFO:teuthology.orchestra.run.smithi092.stderr:512 bytes copied, 0.000196718 s, 2.6 MB/s 2024-09-21T09:41:08.166 DEBUG:teuthology.orchestra.run.smithi092:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-21T09:41:08.222 INFO:tasks.ceph:osd dev map: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-09-21T09:41:08.222 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi092.front.sepia.ceph.com'): {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'}} 2024-09-21T09:41:08.222 INFO:tasks.ceph:Generating config... 2024-09-21T09:41:08.224 INFO:tasks.ceph:[global] mon warn on pool no app = False 2024-09-21T09:41:08.224 INFO:tasks.ceph:[global] ms bind msgr2 = False 2024-09-21T09:41:08.224 INFO:tasks.ceph:[global] bluestore warn on legacy statfs = False 2024-09-21T09:41:08.224 INFO:tasks.ceph:[global] bluestore warn on no per pool omap = False 2024-09-21T09:41:08.224 INFO:tasks.ceph:[global] mon pg warn min per osd = 0 2024-09-21T09:41:08.224 INFO:tasks.ceph:[mgr] client mount timeout = 30 2024-09-21T09:41:08.224 INFO:tasks.ceph:[mgr] debug client = 20 2024-09-21T09:41:08.224 INFO:tasks.ceph:[mgr] debug mgr = 20 2024-09-21T09:41:08.224 INFO:tasks.ceph:[mgr] debug ms = 1 2024-09-21T09:41:08.224 INFO:tasks.ceph:[mon] debug mon = 20 2024-09-21T09:41:08.224 INFO:tasks.ceph:[mon] debug ms = 1 2024-09-21T09:41:08.224 INFO:tasks.ceph:[mon] debug paxos = 20 2024-09-21T09:41:08.224 INFO:tasks.ceph:[mon] mon down mkfs grace = 300 2024-09-21T09:41:08.224 INFO:tasks.ceph:[mon] mon warn on osd down out interval zero = False 2024-09-21T09:41:08.225 INFO:tasks.ceph:[osd] bdev async discard = True 2024-09-21T09:41:08.225 INFO:tasks.ceph:[osd] bdev enable discard = True 2024-09-21T09:41:08.225 INFO:tasks.ceph:[osd] bluestore allocator = bitmap 2024-09-21T09:41:08.225 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2024-09-21T09:41:08.225 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2024-09-21T09:41:08.225 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2024-09-21T09:41:08.225 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2024-09-21T09:41:08.225 INFO:tasks.ceph:[osd] debug ms = 1 2024-09-21T09:41:08.225 INFO:tasks.ceph:[osd] debug osd = 20 2024-09-21T09:41:08.225 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2024-09-21T09:41:08.225 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2024-09-21T09:41:08.225 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2024-09-21T09:41:08.225 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2024-09-21T09:41:08.225 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2024-09-21T09:41:08.225 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2024-09-21T09:41:08.226 INFO:tasks.ceph:Setting up mon.a... 2024-09-21T09:41:08.226 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/ceph.keyring 2024-09-21T09:41:08.305 INFO:teuthology.orchestra.run.smithi092.stdout:creating /etc/ceph/ceph.keyring 2024-09-21T09:41:08.308 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/ceph.keyring 2024-09-21T09:41:08.391 DEBUG:teuthology.orchestra.run.smithi092:> sudo chmod 0644 /etc/ceph/ceph.keyring 2024-09-21T09:41:08.455 DEBUG:tasks.ceph:Ceph mon addresses: [('mon.a', '172.21.15.92'), ('mon.b', '[v2:172.21.15.92:3301,v1:172.21.15.92:6790]'), ('mon.c', '[v2:172.21.15.92:3302,v1:172.21.15.92:6791]')] 2024-09-21T09:41:08.455 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.92,[v2:172.21.15.92:3301,v1:172.21.15.92:6790],[v2:172.21.15.92:3302,v1:172.21.15.92:6791]', 'mon warn on pool no app': False, 'ms bind msgr2': False, 'bluestore warn on legacy statfs': False, 'bluestore warn on no per pool omap': False, 'mon pg warn min per osd': 0}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false', 'client mount timeout': 30, 'debug client': 20}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false', 'mon down mkfs grace': 300, 'mon warn on osd down out interval zero': False}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok'}, 'mon.a': {}, 'mon.b': {}, 'mon.c': {}} 2024-09-21T09:41:08.456 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:41:08.456 DEBUG:teuthology.orchestra.run.smithi092:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2024-09-21T09:41:08.511 DEBUG:teuthology.orchestra.run.smithi092:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.92 --addv b '[v2:172.21.15.92:3301,v1:172.21.15.92:6790]' --addv c '[v2:172.21.15.92:3302,v1:172.21.15.92:6791]' --print /home/ubuntu/cephtest/ceph.monmap 2024-09-21T09:41:08.597 INFO:teuthology.orchestra.run.smithi092.stdout:monmaptool: monmap file /home/ubuntu/cephtest/ceph.monmap 2024-09-21T09:41:08.598 INFO:teuthology.orchestra.run.smithi092.stdout:monmaptool: generated fsid 5f250e6c-4e36-49b1-ab62-d9454369fa03 2024-09-21T09:41:08.598 INFO:teuthology.orchestra.run.smithi092.stdout:setting min_mon_release = quincy 2024-09-21T09:41:08.598 INFO:teuthology.orchestra.run.smithi092.stdout:epoch 0 2024-09-21T09:41:08.598 INFO:teuthology.orchestra.run.smithi092.stdout:fsid 5f250e6c-4e36-49b1-ab62-d9454369fa03 2024-09-21T09:41:08.598 INFO:teuthology.orchestra.run.smithi092.stdout:last_changed 2024-09-21T09:41:08.596905+0000 2024-09-21T09:41:08.598 INFO:teuthology.orchestra.run.smithi092.stdout:created 2024-09-21T09:41:08.596905+0000 2024-09-21T09:41:08.598 INFO:teuthology.orchestra.run.smithi092.stdout:min_mon_release 17 (quincy) 2024-09-21T09:41:08.598 INFO:teuthology.orchestra.run.smithi092.stdout:election_strategy: 1 2024-09-21T09:41:08.598 INFO:teuthology.orchestra.run.smithi092.stdout:0: [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] mon.a 2024-09-21T09:41:08.598 INFO:teuthology.orchestra.run.smithi092.stdout:1: [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] mon.b 2024-09-21T09:41:08.598 INFO:teuthology.orchestra.run.smithi092.stdout:2: [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] mon.c 2024-09-21T09:41:08.598 INFO:teuthology.orchestra.run.smithi092.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/ceph.monmap (3 monitors) 2024-09-21T09:41:08.599 DEBUG:teuthology.orchestra.run.smithi092:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2024-09-21T09:41:08.614 INFO:tasks.ceph:Writing /etc/ceph/ceph.conf for FSID 5f250e6c-4e36-49b1-ab62-d9454369fa03... 2024-09-21T09:41:08.615 DEBUG:teuthology.orchestra.run.smithi092:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-09-21T09:41:08.697 INFO:teuthology.orchestra.run.smithi092.stdout:[global] 2024-09-21T09:41:08.697 INFO:teuthology.orchestra.run.smithi092.stdout: chdir = "" 2024-09-21T09:41:08.697 INFO:teuthology.orchestra.run.smithi092.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-09-21T09:41:08.697 INFO:teuthology.orchestra.run.smithi092.stdout: auth supported = cephx 2024-09-21T09:41:08.698 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.698 INFO:teuthology.orchestra.run.smithi092.stdout: filestore xattr use omap = true 2024-09-21T09:41:08.698 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.698 INFO:teuthology.orchestra.run.smithi092.stdout: mon clock drift allowed = 1.000 2024-09-21T09:41:08.698 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.698 INFO:teuthology.orchestra.run.smithi092.stdout: osd crush chooseleaf type = 0 2024-09-21T09:41:08.698 INFO:teuthology.orchestra.run.smithi092.stdout: auth debug = true 2024-09-21T09:41:08.698 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.698 INFO:teuthology.orchestra.run.smithi092.stdout: ms die on old message = true 2024-09-21T09:41:08.698 INFO:teuthology.orchestra.run.smithi092.stdout: ms die on bug = true 2024-09-21T09:41:08.698 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.698 INFO:teuthology.orchestra.run.smithi092.stdout: mon max pg per osd = 10000 # >= luminous 2024-09-21T09:41:08.699 INFO:teuthology.orchestra.run.smithi092.stdout: mon pg warn max object skew = 0 2024-09-21T09:41:08.699 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.699 INFO:teuthology.orchestra.run.smithi092.stdout: # disable pg_autoscaler by default for new pools 2024-09-21T09:41:08.699 INFO:teuthology.orchestra.run.smithi092.stdout: osd_pool_default_pg_autoscale_mode = off 2024-09-21T09:41:08.699 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.699 INFO:teuthology.orchestra.run.smithi092.stdout: osd pool default size = 2 2024-09-21T09:41:08.699 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.699 INFO:teuthology.orchestra.run.smithi092.stdout: mon osd allow primary affinity = true 2024-09-21T09:41:08.699 INFO:teuthology.orchestra.run.smithi092.stdout: mon osd allow pg remap = true 2024-09-21T09:41:08.699 INFO:teuthology.orchestra.run.smithi092.stdout: mon warn on legacy crush tunables = false 2024-09-21T09:41:08.699 INFO:teuthology.orchestra.run.smithi092.stdout: mon warn on crush straw calc version zero = false 2024-09-21T09:41:08.699 INFO:teuthology.orchestra.run.smithi092.stdout: mon warn on no sortbitwise = false 2024-09-21T09:41:08.699 INFO:teuthology.orchestra.run.smithi092.stdout: mon warn on osd down out interval zero = false 2024-09-21T09:41:08.700 INFO:teuthology.orchestra.run.smithi092.stdout: mon warn on too few osds = false 2024-09-21T09:41:08.700 INFO:teuthology.orchestra.run.smithi092.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-09-21T09:41:08.700 INFO:teuthology.orchestra.run.smithi092.stdout: mon_warn_on_pool_no_redundancy = false 2024-09-21T09:41:08.700 INFO:teuthology.orchestra.run.smithi092.stdout: mon_allow_pool_size_one = true 2024-09-21T09:41:08.700 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.700 INFO:teuthology.orchestra.run.smithi092.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-09-21T09:41:08.700 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.700 INFO:teuthology.orchestra.run.smithi092.stdout: osd default data pool replay window = 5 2024-09-21T09:41:08.700 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.700 INFO:teuthology.orchestra.run.smithi092.stdout: mon allow pool delete = true 2024-09-21T09:41:08.700 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.700 INFO:teuthology.orchestra.run.smithi092.stdout: mon cluster log file level = debug 2024-09-21T09:41:08.700 INFO:teuthology.orchestra.run.smithi092.stdout: debug asserts on shutdown = true 2024-09-21T09:41:08.700 INFO:teuthology.orchestra.run.smithi092.stdout: mon health detail to clog = false 2024-09-21T09:41:08.701 INFO:teuthology.orchestra.run.smithi092.stdout: mon host = "172.21.15.92,[v2:172.21.15.92:3301,v1:172.21.15.92:6790],[v2:172.21.15.92:3302,v1:172.21.15.92:6791]" 2024-09-21T09:41:08.701 INFO:teuthology.orchestra.run.smithi092.stdout: mon warn on pool no app = False 2024-09-21T09:41:08.701 INFO:teuthology.orchestra.run.smithi092.stdout: ms bind msgr2 = False 2024-09-21T09:41:08.701 INFO:teuthology.orchestra.run.smithi092.stdout: bluestore warn on legacy statfs = False 2024-09-21T09:41:08.701 INFO:teuthology.orchestra.run.smithi092.stdout: bluestore warn on no per pool omap = False 2024-09-21T09:41:08.701 INFO:teuthology.orchestra.run.smithi092.stdout: mon pg warn min per osd = 0 2024-09-21T09:41:08.701 INFO:teuthology.orchestra.run.smithi092.stdout: fsid = 5f250e6c-4e36-49b1-ab62-d9454369fa03 2024-09-21T09:41:08.701 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.701 INFO:teuthology.orchestra.run.smithi092.stdout:[osd] 2024-09-21T09:41:08.701 INFO:teuthology.orchestra.run.smithi092.stdout: osd journal size = 100 2024-09-21T09:41:08.701 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.701 INFO:teuthology.orchestra.run.smithi092.stdout: osd scrub load threshold = 5.0 2024-09-21T09:41:08.702 INFO:teuthology.orchestra.run.smithi092.stdout: osd scrub max interval = 600 2024-09-21T09:41:08.702 INFO:teuthology.orchestra.run.smithi092.stdout: osd mclock profile = high_recovery_ops 2024-09-21T09:41:08.702 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.702 INFO:teuthology.orchestra.run.smithi092.stdout: osd recover clone overlap = true 2024-09-21T09:41:08.702 INFO:teuthology.orchestra.run.smithi092.stdout: osd recovery max chunk = 1048576 2024-09-21T09:41:08.702 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.702 INFO:teuthology.orchestra.run.smithi092.stdout: osd debug shutdown = true 2024-09-21T09:41:08.702 INFO:teuthology.orchestra.run.smithi092.stdout: osd debug op order = true 2024-09-21T09:41:08.702 INFO:teuthology.orchestra.run.smithi092.stdout: osd debug verify stray on activate = true 2024-09-21T09:41:08.702 INFO:teuthology.orchestra.run.smithi092.stdout: osd debug trim objects = true 2024-09-21T09:41:08.702 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.703 INFO:teuthology.orchestra.run.smithi092.stdout: osd open classes on start = true 2024-09-21T09:41:08.703 INFO:teuthology.orchestra.run.smithi092.stdout: osd debug pg log writeout = true 2024-09-21T09:41:08.703 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.703 INFO:teuthology.orchestra.run.smithi092.stdout: osd deep scrub update digest min age = 30 2024-09-21T09:41:08.703 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.703 INFO:teuthology.orchestra.run.smithi092.stdout: osd map max advance = 10 2024-09-21T09:41:08.703 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.703 INFO:teuthology.orchestra.run.smithi092.stdout: journal zero on create = true 2024-09-21T09:41:08.703 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.703 INFO:teuthology.orchestra.run.smithi092.stdout: filestore ondisk finisher threads = 3 2024-09-21T09:41:08.703 INFO:teuthology.orchestra.run.smithi092.stdout: filestore apply finisher threads = 3 2024-09-21T09:41:08.703 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.703 INFO:teuthology.orchestra.run.smithi092.stdout: bdev debug aio = true 2024-09-21T09:41:08.703 INFO:teuthology.orchestra.run.smithi092.stdout: osd debug misdirected ops = true 2024-09-21T09:41:08.704 INFO:teuthology.orchestra.run.smithi092.stdout: bdev async discard = True 2024-09-21T09:41:08.704 INFO:teuthology.orchestra.run.smithi092.stdout: bdev enable discard = True 2024-09-21T09:41:08.704 INFO:teuthology.orchestra.run.smithi092.stdout: bluestore allocator = bitmap 2024-09-21T09:41:08.704 INFO:teuthology.orchestra.run.smithi092.stdout: bluestore block size = 96636764160 2024-09-21T09:41:08.704 INFO:teuthology.orchestra.run.smithi092.stdout: bluestore fsck on mount = True 2024-09-21T09:41:08.704 INFO:teuthology.orchestra.run.smithi092.stdout: debug bluefs = 1/20 2024-09-21T09:41:08.704 INFO:teuthology.orchestra.run.smithi092.stdout: debug bluestore = 1/20 2024-09-21T09:41:08.704 INFO:teuthology.orchestra.run.smithi092.stdout: debug ms = 1 2024-09-21T09:41:08.704 INFO:teuthology.orchestra.run.smithi092.stdout: debug osd = 20 2024-09-21T09:41:08.704 INFO:teuthology.orchestra.run.smithi092.stdout: debug rocksdb = 4/10 2024-09-21T09:41:08.704 INFO:teuthology.orchestra.run.smithi092.stdout: mon osd backfillfull_ratio = 0.85 2024-09-21T09:41:08.704 INFO:teuthology.orchestra.run.smithi092.stdout: mon osd full ratio = 0.9 2024-09-21T09:41:08.704 INFO:teuthology.orchestra.run.smithi092.stdout: mon osd nearfull ratio = 0.8 2024-09-21T09:41:08.705 INFO:teuthology.orchestra.run.smithi092.stdout: osd failsafe full ratio = 0.95 2024-09-21T09:41:08.705 INFO:teuthology.orchestra.run.smithi092.stdout: osd objectstore = bluestore 2024-09-21T09:41:08.705 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.705 INFO:teuthology.orchestra.run.smithi092.stdout:[mgr] 2024-09-21T09:41:08.705 INFO:teuthology.orchestra.run.smithi092.stdout: debug ms = 1 2024-09-21T09:41:08.705 INFO:teuthology.orchestra.run.smithi092.stdout: debug mgr = 20 2024-09-21T09:41:08.705 INFO:teuthology.orchestra.run.smithi092.stdout: debug mon = 20 2024-09-21T09:41:08.705 INFO:teuthology.orchestra.run.smithi092.stdout: debug auth = 20 2024-09-21T09:41:08.705 INFO:teuthology.orchestra.run.smithi092.stdout: mon reweight min pgs per osd = 4 2024-09-21T09:41:08.705 INFO:teuthology.orchestra.run.smithi092.stdout: mon reweight min bytes per osd = 10 2024-09-21T09:41:08.705 INFO:teuthology.orchestra.run.smithi092.stdout: mgr/telemetry/nag = false 2024-09-21T09:41:08.705 INFO:teuthology.orchestra.run.smithi092.stdout: client mount timeout = 30 2024-09-21T09:41:08.705 INFO:teuthology.orchestra.run.smithi092.stdout: debug client = 20 2024-09-21T09:41:08.706 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.706 INFO:teuthology.orchestra.run.smithi092.stdout:[mon] 2024-09-21T09:41:08.706 INFO:teuthology.orchestra.run.smithi092.stdout: debug ms = 1 2024-09-21T09:41:08.706 INFO:teuthology.orchestra.run.smithi092.stdout: debug mon = 20 2024-09-21T09:41:08.706 INFO:teuthology.orchestra.run.smithi092.stdout: debug paxos = 20 2024-09-21T09:41:08.706 INFO:teuthology.orchestra.run.smithi092.stdout: debug auth = 20 2024-09-21T09:41:08.706 INFO:teuthology.orchestra.run.smithi092.stdout: mon data avail warn = 5 2024-09-21T09:41:08.706 INFO:teuthology.orchestra.run.smithi092.stdout: mon mgr mkfs grace = 240 2024-09-21T09:41:08.706 INFO:teuthology.orchestra.run.smithi092.stdout: mon reweight min pgs per osd = 4 2024-09-21T09:41:08.706 INFO:teuthology.orchestra.run.smithi092.stdout: mon osd reporter subtree level = osd 2024-09-21T09:41:08.706 INFO:teuthology.orchestra.run.smithi092.stdout: mon osd prime pg temp = true 2024-09-21T09:41:08.706 INFO:teuthology.orchestra.run.smithi092.stdout: mon reweight min bytes per osd = 10 2024-09-21T09:41:08.706 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.707 INFO:teuthology.orchestra.run.smithi092.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-09-21T09:41:08.707 INFO:teuthology.orchestra.run.smithi092.stdout: auth mon ticket ttl = 660 # 11m 2024-09-21T09:41:08.707 INFO:teuthology.orchestra.run.smithi092.stdout: auth service ticket ttl = 240 # 4m 2024-09-21T09:41:08.707 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.707 INFO:teuthology.orchestra.run.smithi092.stdout: # don't complain about insecure global_id in the test suite 2024-09-21T09:41:08.707 INFO:teuthology.orchestra.run.smithi092.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-09-21T09:41:08.707 INFO:teuthology.orchestra.run.smithi092.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-09-21T09:41:08.707 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.707 INFO:teuthology.orchestra.run.smithi092.stdout: # 1m isn't quite enough 2024-09-21T09:41:08.707 INFO:teuthology.orchestra.run.smithi092.stdout: mon_down_mkfs_grace = 2m 2024-09-21T09:41:08.707 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.707 INFO:teuthology.orchestra.run.smithi092.stdout: mon_warn_on_filestore_osds = false 2024-09-21T09:41:08.707 INFO:teuthology.orchestra.run.smithi092.stdout: mon down mkfs grace = 300 2024-09-21T09:41:08.707 INFO:teuthology.orchestra.run.smithi092.stdout: mon warn on osd down out interval zero = False 2024-09-21T09:41:08.708 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:08.708 INFO:teuthology.orchestra.run.smithi092.stdout:[client] 2024-09-21T09:41:08.708 INFO:teuthology.orchestra.run.smithi092.stdout: rgw cache enabled = true 2024-09-21T09:41:08.708 INFO:teuthology.orchestra.run.smithi092.stdout: rgw enable ops log = true 2024-09-21T09:41:08.708 INFO:teuthology.orchestra.run.smithi092.stdout: rgw enable usage log = true 2024-09-21T09:41:08.708 INFO:teuthology.orchestra.run.smithi092.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-09-21T09:41:08.708 INFO:teuthology.orchestra.run.smithi092.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-09-21T09:41:08.708 INFO:teuthology.orchestra.run.smithi092.stdout:[mon.a] 2024-09-21T09:41:08.708 INFO:teuthology.orchestra.run.smithi092.stdout:[mon.b] 2024-09-21T09:41:08.708 INFO:teuthology.orchestra.run.smithi092.stdout:[mon.c] 2024-09-21T09:41:08.709 INFO:tasks.ceph:Creating admin key on mon.a... 2024-09-21T09:41:08.710 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/ceph.keyring 2024-09-21T09:41:08.803 INFO:tasks.ceph:Copying monmap to all nodes... 2024-09-21T09:41:08.804 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:41:08.804 DEBUG:teuthology.orchestra.run.smithi092:> dd if=/etc/ceph/ceph.keyring of=/dev/stdout 2024-09-21T09:41:08.819 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:41:08.819 DEBUG:teuthology.orchestra.run.smithi092:> dd if=/home/ubuntu/cephtest/ceph.monmap of=/dev/stdout 2024-09-21T09:41:08.874 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi092.front.sepia.ceph.com 2024-09-21T09:41:08.875 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:41:08.875 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd of=/etc/ceph/ceph.keyring 2024-09-21T09:41:08.875 DEBUG:teuthology.orchestra.run.smithi092:> sudo chmod 0644 /etc/ceph/ceph.keyring 2024-09-21T09:41:08.949 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:41:08.949 DEBUG:teuthology.orchestra.run.smithi092:> dd of=/home/ubuntu/cephtest/ceph.monmap 2024-09-21T09:41:09.005 INFO:tasks.ceph:Setting up mon nodes... 2024-09-21T09:41:09.005 INFO:tasks.ceph:Setting up mgr nodes... 2024-09-21T09:41:09.005 DEBUG:teuthology.orchestra.run.smithi092:> sudo mkdir -p /var/lib/ceph/mgr/ceph-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/ceph-x/keyring 2024-09-21T09:41:09.095 INFO:teuthology.orchestra.run.smithi092.stdout:creating /var/lib/ceph/mgr/ceph-x/keyring 2024-09-21T09:41:09.098 DEBUG:teuthology.orchestra.run.smithi092:> sudo mkdir -p /var/lib/ceph/mgr/ceph-y && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.y /var/lib/ceph/mgr/ceph-y/keyring 2024-09-21T09:41:09.148 INFO:teuthology.orchestra.run.smithi092.stdout:creating /var/lib/ceph/mgr/ceph-y/keyring 2024-09-21T09:41:09.151 INFO:tasks.ceph:Setting up mds nodes... 2024-09-21T09:41:09.151 INFO:tasks.ceph_client:Setting up client nodes... 2024-09-21T09:41:09.151 INFO:tasks.ceph:Running mkfs on osd nodes... 2024-09-21T09:41:09.151 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi092.front.sepia.ceph.com'): {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'}} 2024-09-21T09:41:09.151 DEBUG:teuthology.orchestra.run.smithi092:> sudo mkdir -p /var/lib/ceph/osd/ceph-0 2024-09-21T09:41:09.175 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-09-21T09:41:09.176 INFO:tasks.ceph:role: osd.0 2024-09-21T09:41:09.176 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi092.front.sepia.ceph.com 2024-09-21T09:41:09.176 DEBUG:teuthology.orchestra.run.smithi092:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2024-09-21T09:41:09.247 INFO:teuthology.orchestra.run.smithi092.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2024-09-21T09:41:09.247 INFO:teuthology.orchestra.run.smithi092.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-21T09:41:09.247 INFO:teuthology.orchestra.run.smithi092.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-21T09:41:09.247 INFO:teuthology.orchestra.run.smithi092.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-21T09:41:09.247 INFO:teuthology.orchestra.run.smithi092.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-21T09:41:09.247 INFO:teuthology.orchestra.run.smithi092.stdout: = sunit=0 swidth=0 blks 2024-09-21T09:41:09.248 INFO:teuthology.orchestra.run.smithi092.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-21T09:41:09.248 INFO:teuthology.orchestra.run.smithi092.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-21T09:41:09.248 INFO:teuthology.orchestra.run.smithi092.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-21T09:41:09.248 INFO:teuthology.orchestra.run.smithi092.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-21T09:41:09.762 INFO:teuthology.orchestra.run.smithi092.stdout:Discarding blocks...Done. 2024-09-21T09:41:09.764 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi092.front.sepia.ceph.com -o noatime 2024-09-21T09:41:09.764 DEBUG:teuthology.orchestra.run.smithi092:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-0 2024-09-21T09:41:09.904 DEBUG:teuthology.orchestra.run.smithi092:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-0 2024-09-21T09:41:09.936 DEBUG:teuthology.orchestra.run.smithi092:> sudo mkdir -p /var/lib/ceph/osd/ceph-1 2024-09-21T09:41:10.000 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-09-21T09:41:10.000 INFO:tasks.ceph:role: osd.1 2024-09-21T09:41:10.000 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi092.front.sepia.ceph.com 2024-09-21T09:41:10.000 DEBUG:teuthology.orchestra.run.smithi092:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2024-09-21T09:41:10.073 INFO:teuthology.orchestra.run.smithi092.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2024-09-21T09:41:10.073 INFO:teuthology.orchestra.run.smithi092.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-21T09:41:10.073 INFO:teuthology.orchestra.run.smithi092.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-21T09:41:10.073 INFO:teuthology.orchestra.run.smithi092.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-21T09:41:10.073 INFO:teuthology.orchestra.run.smithi092.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-21T09:41:10.073 INFO:teuthology.orchestra.run.smithi092.stdout: = sunit=0 swidth=0 blks 2024-09-21T09:41:10.073 INFO:teuthology.orchestra.run.smithi092.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-21T09:41:10.073 INFO:teuthology.orchestra.run.smithi092.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-21T09:41:10.073 INFO:teuthology.orchestra.run.smithi092.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-21T09:41:10.073 INFO:teuthology.orchestra.run.smithi092.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-21T09:41:10.521 INFO:teuthology.orchestra.run.smithi092.stdout:Discarding blocks...Done. 2024-09-21T09:41:10.523 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi092.front.sepia.ceph.com -o noatime 2024-09-21T09:41:10.523 DEBUG:teuthology.orchestra.run.smithi092:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-1 2024-09-21T09:41:10.609 DEBUG:teuthology.orchestra.run.smithi092:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-1 2024-09-21T09:41:10.639 DEBUG:teuthology.orchestra.run.smithi092:> sudo mkdir -p /var/lib/ceph/osd/ceph-2 2024-09-21T09:41:10.704 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-09-21T09:41:10.704 INFO:tasks.ceph:role: osd.2 2024-09-21T09:41:10.704 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi092.front.sepia.ceph.com 2024-09-21T09:41:10.704 DEBUG:teuthology.orchestra.run.smithi092:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2024-09-21T09:41:10.777 INFO:teuthology.orchestra.run.smithi092.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2024-09-21T09:41:10.777 INFO:teuthology.orchestra.run.smithi092.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-21T09:41:10.777 INFO:teuthology.orchestra.run.smithi092.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-21T09:41:10.777 INFO:teuthology.orchestra.run.smithi092.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-21T09:41:10.777 INFO:teuthology.orchestra.run.smithi092.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-21T09:41:10.778 INFO:teuthology.orchestra.run.smithi092.stdout: = sunit=0 swidth=0 blks 2024-09-21T09:41:10.778 INFO:teuthology.orchestra.run.smithi092.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-21T09:41:10.778 INFO:teuthology.orchestra.run.smithi092.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-21T09:41:10.778 INFO:teuthology.orchestra.run.smithi092.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-21T09:41:10.778 INFO:teuthology.orchestra.run.smithi092.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-21T09:41:11.264 INFO:teuthology.orchestra.run.smithi092.stdout:Discarding blocks...Done. 2024-09-21T09:41:11.266 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi092.front.sepia.ceph.com -o noatime 2024-09-21T09:41:11.266 DEBUG:teuthology.orchestra.run.smithi092:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-2 2024-09-21T09:41:11.419 DEBUG:teuthology.orchestra.run.smithi092:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-2 2024-09-21T09:41:11.449 DEBUG:teuthology.orchestra.run.smithi092:> sudo mkdir -p /var/lib/ceph/osd/ceph-3 2024-09-21T09:41:11.514 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-09-21T09:41:11.514 INFO:tasks.ceph:role: osd.3 2024-09-21T09:41:11.514 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_4 on ubuntu@smithi092.front.sepia.ceph.com 2024-09-21T09:41:11.514 DEBUG:teuthology.orchestra.run.smithi092:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_4 2024-09-21T09:41:11.586 INFO:teuthology.orchestra.run.smithi092.stdout:meta-data=/dev/vg_nvme/lv_4 isize=2048 agcount=4, agsize=5859072 blks 2024-09-21T09:41:11.586 INFO:teuthology.orchestra.run.smithi092.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-21T09:41:11.586 INFO:teuthology.orchestra.run.smithi092.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-21T09:41:11.586 INFO:teuthology.orchestra.run.smithi092.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-21T09:41:11.586 INFO:teuthology.orchestra.run.smithi092.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-21T09:41:11.586 INFO:teuthology.orchestra.run.smithi092.stdout: = sunit=0 swidth=0 blks 2024-09-21T09:41:11.586 INFO:teuthology.orchestra.run.smithi092.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-21T09:41:11.586 INFO:teuthology.orchestra.run.smithi092.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-21T09:41:11.587 INFO:teuthology.orchestra.run.smithi092.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-21T09:41:11.587 INFO:teuthology.orchestra.run.smithi092.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-21T09:41:12.110 INFO:teuthology.orchestra.run.smithi092.stdout:Discarding blocks...Done. 2024-09-21T09:41:12.112 INFO:tasks.ceph:mount /dev/vg_nvme/lv_4 on ubuntu@smithi092.front.sepia.ceph.com -o noatime 2024-09-21T09:41:12.112 DEBUG:teuthology.orchestra.run.smithi092:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-3 2024-09-21T09:41:12.266 DEBUG:teuthology.orchestra.run.smithi092:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-3 2024-09-21T09:41:12.297 DEBUG:teuthology.orchestra.run.smithi092:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-09-21T09:41:12.398 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:12.396+0000 7fec2cfc5640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-0/keyring: can't open /var/lib/ceph/osd/ceph-0/keyring: (2) No such file or directory 2024-09-21T09:41:12.399 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:12.397+0000 7fec2cfc5640 -1 created new key in keyring /var/lib/ceph/osd/ceph-0/keyring 2024-09-21T09:41:12.399 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:12.397+0000 7fec2cfc5640 -1 bdev(0x55e7cd9ad800 /var/lib/ceph/osd/ceph-0/block) open stat got: (1) Operation not permitted 2024-09-21T09:41:12.399 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:12.397+0000 7fec2cfc5640 -1 bluestore(/var/lib/ceph/osd/ceph-0) _read_fsid unparsable uuid 2024-09-21T09:41:16.304 DEBUG:teuthology.orchestra.run.smithi092:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T09:41:16.331 DEBUG:teuthology.orchestra.run.smithi092:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-09-21T09:41:16.420 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:16.418+0000 7f1227616640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-1/keyring: can't open /var/lib/ceph/osd/ceph-1/keyring: (2) No such file or directory 2024-09-21T09:41:16.420 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:16.418+0000 7f1227616640 -1 created new key in keyring /var/lib/ceph/osd/ceph-1/keyring 2024-09-21T09:41:16.420 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:16.418+0000 7f1227616640 -1 bdev(0x563f02173800 /var/lib/ceph/osd/ceph-1/block) open stat got: (1) Operation not permitted 2024-09-21T09:41:16.420 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:16.418+0000 7f1227616640 -1 bluestore(/var/lib/ceph/osd/ceph-1) _read_fsid unparsable uuid 2024-09-21T09:41:20.073 DEBUG:teuthology.orchestra.run.smithi092:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T09:41:20.102 DEBUG:teuthology.orchestra.run.smithi092:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-09-21T09:41:20.192 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:20.189+0000 7f7dd500b640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-2/keyring: can't open /var/lib/ceph/osd/ceph-2/keyring: (2) No such file or directory 2024-09-21T09:41:20.192 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:20.190+0000 7f7dd500b640 -1 created new key in keyring /var/lib/ceph/osd/ceph-2/keyring 2024-09-21T09:41:20.192 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:20.190+0000 7f7dd500b640 -1 bdev(0x55b59d465800 /var/lib/ceph/osd/ceph-2/block) open stat got: (1) Operation not permitted 2024-09-21T09:41:20.192 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:20.190+0000 7f7dd500b640 -1 bluestore(/var/lib/ceph/osd/ceph-2) _read_fsid unparsable uuid 2024-09-21T09:41:24.100 DEBUG:teuthology.orchestra.run.smithi092:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T09:41:24.130 DEBUG:teuthology.orchestra.run.smithi092:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 3 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-09-21T09:41:24.223 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:24.221+0000 7fd9f2c16640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-3/keyring: can't open /var/lib/ceph/osd/ceph-3/keyring: (2) No such file or directory 2024-09-21T09:41:24.224 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:24.222+0000 7fd9f2c16640 -1 created new key in keyring /var/lib/ceph/osd/ceph-3/keyring 2024-09-21T09:41:24.224 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:24.222+0000 7fd9f2c16640 -1 bdev(0x55d9c0b97800 /var/lib/ceph/osd/ceph-3/block) open stat got: (1) Operation not permitted 2024-09-21T09:41:24.224 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:24.222+0000 7fd9f2c16640 -1 bluestore(/var/lib/ceph/osd/ceph-3) _read_fsid unparsable uuid 2024-09-21T09:41:27.871 DEBUG:teuthology.orchestra.run.smithi092:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T09:41:27.900 INFO:tasks.ceph:Reading keys from all nodes... 2024-09-21T09:41:27.900 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:41:27.901 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd if=/var/lib/ceph/mgr/ceph-x/keyring of=/dev/stdout 2024-09-21T09:41:27.968 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:41:27.969 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd if=/var/lib/ceph/mgr/ceph-y/keyring of=/dev/stdout 2024-09-21T09:41:28.035 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:41:28.035 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd if=/var/lib/ceph/osd/ceph-0/keyring of=/dev/stdout 2024-09-21T09:41:28.103 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:41:28.103 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd if=/var/lib/ceph/osd/ceph-1/keyring of=/dev/stdout 2024-09-21T09:41:28.170 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:41:28.170 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd if=/var/lib/ceph/osd/ceph-2/keyring of=/dev/stdout 2024-09-21T09:41:28.234 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:41:28.234 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd if=/var/lib/ceph/osd/ceph-3/keyring of=/dev/stdout 2024-09-21T09:41:28.299 INFO:tasks.ceph:Adding keys to all mons... 2024-09-21T09:41:28.299 DEBUG:teuthology.orchestra.run.smithi092:> sudo tee -a /etc/ceph/ceph.keyring 2024-09-21T09:41:28.364 INFO:teuthology.orchestra.run.smithi092.stdout:[mgr.x] 2024-09-21T09:41:28.364 INFO:teuthology.orchestra.run.smithi092.stdout: key = AQC1lO5m7lCrBRAAlHRsv63DOri10XcGeaqLXQ== 2024-09-21T09:41:28.364 INFO:teuthology.orchestra.run.smithi092.stdout:[mgr.y] 2024-09-21T09:41:28.364 INFO:teuthology.orchestra.run.smithi092.stdout: key = AQC1lO5m7FDPCBAArtaoj+ERJx8RyJP/5cua5Q== 2024-09-21T09:41:28.364 INFO:teuthology.orchestra.run.smithi092.stdout:[osd.0] 2024-09-21T09:41:28.364 INFO:teuthology.orchestra.run.smithi092.stdout: key = AQC4lO5mDdCzFxAAsBFngAnOOhYC9YJ++7d1yQ== 2024-09-21T09:41:28.364 INFO:teuthology.orchestra.run.smithi092.stdout:[osd.1] 2024-09-21T09:41:28.364 INFO:teuthology.orchestra.run.smithi092.stdout: key = AQC8lO5mI/H6GBAAlOs0wgukdazQEJ+Jnk5HdA== 2024-09-21T09:41:28.364 INFO:teuthology.orchestra.run.smithi092.stdout:[osd.2] 2024-09-21T09:41:28.364 INFO:teuthology.orchestra.run.smithi092.stdout: key = AQDAlO5m3MFdCxAAfd+PiJqQhb0tC19bni8kxA== 2024-09-21T09:41:28.365 INFO:teuthology.orchestra.run.smithi092.stdout:[osd.3] 2024-09-21T09:41:28.365 INFO:teuthology.orchestra.run.smithi092.stdout: key = AQDElO5mKq9GDRAAY4jCxpr8i+BUxsqBeQ5TwA== 2024-09-21T09:41:28.365 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-09-21T09:41:28.457 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.y --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-09-21T09:41:28.509 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-21T09:41:28.560 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-21T09:41:28.617 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-21T09:41:28.668 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.3 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-21T09:41:28.718 INFO:tasks.ceph:Running mkfs on mon nodes... 2024-09-21T09:41:28.718 DEBUG:teuthology.orchestra.run.smithi092:> sudo mkdir -p /var/lib/ceph/mon/ceph-a 2024-09-21T09:41:28.743 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i a --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2024-09-21T09:41:28.840 DEBUG:teuthology.orchestra.run.smithi092:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-a 2024-09-21T09:41:28.864 DEBUG:teuthology.orchestra.run.smithi092:> sudo mkdir -p /var/lib/ceph/mon/ceph-b 2024-09-21T09:41:28.929 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i b --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2024-09-21T09:41:29.025 DEBUG:teuthology.orchestra.run.smithi092:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-b 2024-09-21T09:41:29.050 DEBUG:teuthology.orchestra.run.smithi092:> sudo mkdir -p /var/lib/ceph/mon/ceph-c 2024-09-21T09:41:29.115 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i c --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2024-09-21T09:41:29.209 DEBUG:teuthology.orchestra.run.smithi092:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-c 2024-09-21T09:41:29.234 DEBUG:teuthology.orchestra.run.smithi092:> rm -- /home/ubuntu/cephtest/ceph.monmap 2024-09-21T09:41:29.290 INFO:tasks.ceph:Starting mon daemons in cluster ceph... 2024-09-21T09:41:29.290 INFO:tasks.ceph.mon.a:Restarting daemon 2024-09-21T09:41:29.290 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i a 2024-09-21T09:41:29.332 INFO:tasks.ceph.mon.a:Started 2024-09-21T09:41:29.332 INFO:tasks.ceph.mon.b:Restarting daemon 2024-09-21T09:41:29.333 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i b 2024-09-21T09:41:29.334 INFO:tasks.ceph.mon.b:Started 2024-09-21T09:41:29.335 INFO:tasks.ceph.mon.c:Restarting daemon 2024-09-21T09:41:29.335 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i c 2024-09-21T09:41:29.336 INFO:tasks.ceph.mon.c:Started 2024-09-21T09:41:29.337 INFO:tasks.ceph:Starting mgr daemons in cluster ceph... 2024-09-21T09:41:29.337 INFO:tasks.ceph.mgr.x:Restarting daemon 2024-09-21T09:41:29.337 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i x 2024-09-21T09:41:29.339 INFO:tasks.ceph.mgr.x:Started 2024-09-21T09:41:29.339 INFO:tasks.ceph.mgr.y:Restarting daemon 2024-09-21T09:41:29.339 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i y 2024-09-21T09:41:29.341 INFO:tasks.ceph.mgr.y:Started 2024-09-21T09:41:29.341 INFO:tasks.ceph:setting config [client] client_mount_timeout = 600 2024-09-21T09:41:29.341 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph config set client client_mount_timeout 600 2024-09-21T09:41:29.343 INFO:tasks.ceph:setting config [client] debug_client = 20 2024-09-21T09:41:29.343 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph config set client debug_client 20 2024-09-21T09:41:29.389 INFO:tasks.ceph:setting config [client] debug_ms = 1 2024-09-21T09:41:29.389 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph config set client debug_ms 1 2024-09-21T09:41:29.431 INFO:tasks.ceph:setting config [client] rados_mon_op_timeout = 900 2024-09-21T09:41:29.431 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph config set client rados_mon_op_timeout 900 2024-09-21T09:41:29.476 INFO:tasks.ceph:setting config [client] rados_osd_op_timeout = 900 2024-09-21T09:41:29.502 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph config set client rados_osd_op_timeout 900 2024-09-21T09:41:29.545 INFO:tasks.ceph:setting config [mds] debug_mds = 20 2024-09-21T09:41:29.545 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph config set mds debug_mds 20 2024-09-21T09:41:29.587 INFO:tasks.ceph:setting config [mds] debug_mds_balancer = 20 2024-09-21T09:41:29.587 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph config set mds debug_mds_balancer 20 2024-09-21T09:41:29.630 INFO:tasks.ceph:setting config [mds] debug_ms = 2 2024-09-21T09:41:29.630 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph config set mds debug_ms 2 2024-09-21T09:41:29.675 INFO:tasks.ceph:setting config [mds] mds_debug_frag = True 2024-09-21T09:41:29.675 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph config set mds mds_debug_frag True 2024-09-21T09:41:29.717 INFO:tasks.ceph:setting config [mds] mds_debug_scatterstat = True 2024-09-21T09:41:29.717 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph config set mds mds_debug_scatterstat True 2024-09-21T09:41:29.761 INFO:tasks.ceph:setting config [mds] mds_op_complaint_time = 180 2024-09-21T09:41:29.761 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph config set mds mds_op_complaint_time 180 2024-09-21T09:41:29.805 INFO:tasks.ceph:setting config [mds] mds_verify_scatter = True 2024-09-21T09:41:29.805 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph config set mds mds_verify_scatter True 2024-09-21T09:41:29.849 INFO:tasks.ceph:setting config [mds] osd_op_complaint_time = 180 2024-09-21T09:41:29.849 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph config set mds osd_op_complaint_time 180 2024-09-21T09:41:29.892 INFO:tasks.ceph:setting config [mds] rados_mon_op_timeout = 900 2024-09-21T09:41:29.892 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph config set mds rados_mon_op_timeout 900 2024-09-21T09:41:29.935 INFO:tasks.ceph:setting config [mds] rados_osd_op_timeout = 900 2024-09-21T09:41:29.935 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph config set mds rados_osd_op_timeout 900 2024-09-21T09:41:29.978 INFO:tasks.ceph:setting config [mon] mon_op_complaint_time = 120 2024-09-21T09:41:29.979 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph config set mon mon_op_complaint_time 120 2024-09-21T09:41:30.021 INFO:tasks.ceph:setting config [mon] mon_warn_on_pool_no_app = False 2024-09-21T09:41:30.021 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph config set mon mon_warn_on_pool_no_app False 2024-09-21T09:41:30.064 INFO:tasks.ceph:setting config [osd] osd_op_complaint_time = 180 2024-09-21T09:41:30.064 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph config set osd osd_op_complaint_time 180 2024-09-21T09:41:30.107 DEBUG:tasks.ceph:set 18 configs 2024-09-21T09:41:30.107 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'client_mount_timeout', '600'], check_status=True, hostname='smithi092') 2024-09-21T09:41:35.558 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:35.555+0000 7ffbab4a0040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:41:35.563 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:35.561+0000 7fdc77016040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:41:35.582 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'debug_client', '20'], check_status=True, hostname='smithi092') 2024-09-21T09:41:35.582 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'debug_ms', '1'], check_status=True, hostname='smithi092') 2024-09-21T09:41:35.602 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:35.600+0000 7ffbab4a0040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:41:35.607 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:35.605+0000 7fdc77016040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:41:35.618 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.615+0000 7f80cc878640 1 -- 172.21.15.92:0/3021254900 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f80c809b520 msgr2=0x7f80c8130e70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.618 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.615+0000 7f80cc878640 1 --2- 172.21.15.92:0/3021254900 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f80c809b520 0x7f80c8130e70 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f80c0005fd0 tx=0x7f80c0013ea0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.618 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.615+0000 7f80cc878640 1 -- 172.21.15.92:0/3021254900 shutdown_connections 2024-09-21T09:41:35.618 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.615+0000 7f80cc878640 1 --2- 172.21.15.92:0/3021254900 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f80c809b520 0x7f80c8130e70 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.618 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.615+0000 7f80cc878640 1 --2- 172.21.15.92:0/3021254900 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f80c809ac00 0x7f80c8130930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.618 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.615+0000 7f80cc878640 1 --2- 172.21.15.92:0/3021254900 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f80a80036d0 0x7f80c81303f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.618 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.615+0000 7f80cc878640 1 -- 172.21.15.92:0/3021254900 >> 172.21.15.92:0/3021254900 conn(0x7f80c809e830 msgr2=0x7f80c812d900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:35.618 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.615+0000 7f80cc878640 1 -- 172.21.15.92:0/3021254900 shutdown_connections 2024-09-21T09:41:35.618 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.615+0000 7f80cc878640 1 -- 172.21.15.92:0/3021254900 wait complete. 2024-09-21T09:41:35.628 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'rados_mon_op_timeout', '900'], check_status=True, hostname='smithi092') 2024-09-21T09:41:35.629 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'rados_osd_op_timeout', '900'], check_status=True, hostname='smithi092') 2024-09-21T09:41:35.645 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.642+0000 7f4f9fd83640 1 -- 172.21.15.92:0/3018282679 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4f9812c770 msgr2=0x7f4f7c00aa80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.645 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.642+0000 7f4f9fd83640 1 --2- 172.21.15.92:0/3018282679 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4f9812c770 0x7f4f7c00aa80 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f4f88005ce0 tx=0x7f4f8802bc50 comp rx=0 tx=0).stop 2024-09-21T09:41:35.645 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.642+0000 7f4f9fd83640 1 -- 172.21.15.92:0/3018282679 shutdown_connections 2024-09-21T09:41:35.645 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.642+0000 7f4f9fd83640 1 --2- 172.21.15.92:0/3018282679 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4f9812c770 0x7f4f7c00aa80 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.645 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.642+0000 7f4f9fd83640 1 --2- 172.21.15.92:0/3018282679 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4f981316f0 0x7f4f7c008510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.645 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.642+0000 7f4f9fd83640 1 --2- 172.21.15.92:0/3018282679 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4f7c003670 0x7f4f7c005b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.645 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.642+0000 7f4f9fd83640 1 -- 172.21.15.92:0/3018282679 >> 172.21.15.92:0/3018282679 conn(0x7f4f9809d070 msgr2=0x7f4f98099540 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:35.645 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.642+0000 7f4f9fd83640 1 -- 172.21.15.92:0/3018282679 shutdown_connections 2024-09-21T09:41:35.646 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.642+0000 7f4f9fd83640 1 -- 172.21.15.92:0/3018282679 wait complete. 2024-09-21T09:41:35.647 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.643+0000 7f4f9fd83640 1 Processor -- start 2024-09-21T09:41:35.647 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.643+0000 7f4f9fd83640 1 -- start start 2024-09-21T09:41:35.648 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.643+0000 7f4f9fd83640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4f7c003670 0x7f4f980912e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:35.648 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.643+0000 7f4f9fd83640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4f9812bc80 0x7f4f98091820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:35.648 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.643+0000 7f4f9fd83640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4f9812c770 0x7f4f98091d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:35.648 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.643+0000 7f4f9fd83640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f4f98096230 con 0x7f4f9812c770 2024-09-21T09:41:35.648 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.643+0000 7f4f9fd83640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f4f980960b0 con 0x7f4f9812bc80 2024-09-21T09:41:35.648 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.643+0000 7f4f9fd83640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f4f980963b0 con 0x7f4f7c003670 2024-09-21T09:41:35.648 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.644+0000 7f4f9d31e640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4f9812bc80 0x7f4f98091820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:35.648 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.644+0000 7f4f9d31e640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4f9812bc80 0x7f4f98091820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:44986/0 (socket says 172.21.15.92:44986) 2024-09-21T09:41:35.648 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.644+0000 7f4f9d31e640 1 -- 172.21.15.92:0/1353499685 learned_addr learned my addr 172.21.15.92:0/1353499685 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:35.648 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.644+0000 7f4f9db1f640 1 --2- 172.21.15.92:0/1353499685 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4f7c003670 0x7f4f980912e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:35.648 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.644+0000 7f4f9e320640 1 --2- 172.21.15.92:0/1353499685 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4f9812c770 0x7f4f98091d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:35.648 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.644+0000 7f4f9d31e640 1 -- 172.21.15.92:0/1353499685 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4f7c003670 msgr2=0x7f4f980912e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.648 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.644+0000 7f4f9d31e640 1 --2- 172.21.15.92:0/1353499685 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4f7c003670 0x7f4f980912e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.648 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.644+0000 7f4f9d31e640 1 -- 172.21.15.92:0/1353499685 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4f9812c770 msgr2=0x7f4f98091d60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.649 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.644+0000 7f4f9d31e640 1 --2- 172.21.15.92:0/1353499685 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4f9812c770 0x7f4f98091d60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.649 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.644+0000 7f4f9d31e640 1 -- 172.21.15.92:0/1353499685 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4f98090470 con 0x7f4f9812bc80 2024-09-21T09:41:35.649 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.644+0000 7f4f9db1f640 1 --2- 172.21.15.92:0/1353499685 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4f7c003670 0x7f4f980912e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:35.649 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.644+0000 7f4f9e320640 1 --2- 172.21.15.92:0/1353499685 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4f9812c770 0x7f4f98091d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:35.649 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.644+0000 7f4f9d31e640 1 --2- 172.21.15.92:0/1353499685 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4f9812bc80 0x7f4f98091820 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f4f88011cf0 tx=0x7f4f88005ce0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:35.649 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.644+0000 7f4f8e7fc640 1 -- 172.21.15.92:0/1353499685 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f88032040 con 0x7f4f9812bc80 2024-09-21T09:41:35.649 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.644+0000 7f4f8e7fc640 1 -- 172.21.15.92:0/1353499685 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(4 keys) ==== 110+0+0 (secure 0 0 0) 0x7f4f8802ce50 con 0x7f4f9812bc80 2024-09-21T09:41:35.649 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.644+0000 7f4f9fd83640 1 -- 172.21.15.92:0/1353499685 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4f9808fa70 con 0x7f4f9812bc80 2024-09-21T09:41:35.649 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.644+0000 7f4f8e7fc640 1 -- 172.21.15.92:0/1353499685 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f88036d70 con 0x7f4f9812bc80 2024-09-21T09:41:35.649 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.644+0000 7f4f9fd83640 1 -- 172.21.15.92:0/1353499685 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f4f98092d00 con 0x7f4f9812bc80 2024-09-21T09:41:35.649 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.645+0000 7f4f8e7fc640 1 -- 172.21.15.92:0/1353499685 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f4f8803f640 con 0x7f4f9812bc80 2024-09-21T09:41:35.649 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.645+0000 7f4f8e7fc640 1 -- 172.21.15.92:0/1353499685 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f4f88051570 con 0x7f4f9812bc80 2024-09-21T09:41:35.650 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.645+0000 7f4f9fd83640 1 -- 172.21.15.92:0/1353499685 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4f98096880 con 0x7f4f9812bc80 2024-09-21T09:41:35.652 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.649+0000 7f4f8e7fc640 1 -- 172.21.15.92:0/1353499685 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7f4f98090470 con 0x7f4f9812bc80 2024-09-21T09:41:35.660 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:35.658+0000 7ffbab4a0040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:41:35.662 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:35.660+0000 7fdc77016040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:41:35.714 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.711+0000 7f4f9fd83640 1 -- 172.21.15.92:0/1353499685 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command([{prefix=config set, name=rados_osd_op_timeout}] v 0) -- 0x7f4f98075950 con 0x7f4f9812bc80 2024-09-21T09:41:35.717 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.714+0000 7f4f8e7fc640 1 -- 172.21.15.92:0/1353499685 <== mon.1 v2:172.21.15.92:3301/0 7 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f4f8803fac0 con 0x7f4f9812bc80 2024-09-21T09:41:35.717 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.714+0000 7f4f8e7fc640 1 -- 172.21.15.92:0/1353499685 <== mon.1 v2:172.21.15.92:3301/0 8 ==== mon_command_ack([{prefix=config set, name=rados_osd_op_timeout}]=0 v10)=0 v10) ==== 123+0+0 (secure 0 0 0) 0x7f4f8803fd90 con 0x7f4f9812bc80 2024-09-21T09:41:35.717 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.715+0000 7f4f9fd83640 1 -- 172.21.15.92:0/1353499685 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4f9812bc80 msgr2=0x7f4f98091820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.718 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.715+0000 7f4f9fd83640 1 --2- 172.21.15.92:0/1353499685 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4f9812bc80 0x7f4f98091820 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f4f88011cf0 tx=0x7f4f88005ce0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.718 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.715+0000 7f4f9fd83640 1 -- 172.21.15.92:0/1353499685 shutdown_connections 2024-09-21T09:41:35.718 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.715+0000 7f4f9fd83640 1 --2- 172.21.15.92:0/1353499685 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4f9812c770 0x7f4f98091d60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.718 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.715+0000 7f4f9fd83640 1 --2- 172.21.15.92:0/1353499685 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4f9812bc80 0x7f4f98091820 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.718 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.715+0000 7f4f9fd83640 1 --2- 172.21.15.92:0/1353499685 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4f7c003670 0x7f4f980912e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.718 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.715+0000 7f4f9fd83640 1 -- 172.21.15.92:0/1353499685 >> 172.21.15.92:0/1353499685 conn(0x7f4f9809d070 msgr2=0x7f4f980956c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:35.718 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.715+0000 7f4f9fd83640 1 -- 172.21.15.92:0/1353499685 shutdown_connections 2024-09-21T09:41:35.718 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.715+0000 7f4f9fd83640 1 -- 172.21.15.92:0/1353499685 wait complete. 2024-09-21T09:41:35.728 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'debug_mds', '20'], check_status=True, hostname='smithi092') 2024-09-21T09:41:35.728 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'debug_mds_balancer', '20'], check_status=True, hostname='smithi092') 2024-09-21T09:41:35.735 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.732+0000 7f209e12c640 1 -- 172.21.15.92:0/2246248163 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2098075330 msgr2=0x7f208400a9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.735 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.732+0000 7f209e12c640 1 --2- 172.21.15.92:0/2246248163 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2098075330 0x7f208400a9c0 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f20980698a0 tx=0x7f209002bb40 comp rx=0 tx=0).stop 2024-09-21T09:41:35.735 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.732+0000 7f209e12c640 1 -- 172.21.15.92:0/2246248163 shutdown_connections 2024-09-21T09:41:35.735 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.732+0000 7f209e12c640 1 --2- 172.21.15.92:0/2246248163 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2098075330 0x7f208400a9c0 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.735 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.732+0000 7f209e12c640 1 --2- 172.21.15.92:0/2246248163 >> v2:172.21.15.92:3300/0 conn(0x7f2098131390 0x7f20840083a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.735 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.732+0000 7f209e12c640 1 --2- 172.21.15.92:0/2246248163 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2084003670 0x7f2084005ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.735 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.732+0000 7f209e12c640 1 -- 172.21.15.92:0/2246248163 >> 172.21.15.92:0/2246248163 conn(0x7f209809cc90 msgr2=0x7f209809d0a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:35.735 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.732+0000 7f209e12c640 1 -- 172.21.15.92:0/2246248163 shutdown_connections 2024-09-21T09:41:35.735 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.732+0000 7f209e12c640 1 -- 172.21.15.92:0/2246248163 wait complete. 2024-09-21T09:41:35.738 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.733+0000 7f209e12c640 1 Processor -- start 2024-09-21T09:41:35.738 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.733+0000 7f209e12c640 1 -- start start 2024-09-21T09:41:35.738 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.733+0000 7f209e12c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2084003670 0x7f209812e210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:35.738 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.733+0000 7f209e12c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2098075330 0x7f209812e750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:35.738 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.733+0000 7f209e12c640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2098096500 0x7f2098093f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:35.738 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.733+0000 7f209e12c640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f209812fa80 con 0x7f2084003670 2024-09-21T09:41:35.738 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.733+0000 7f209e12c640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f209812f900 con 0x7f2098075330 2024-09-21T09:41:35.738 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.733+0000 7f209e12c640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f209812fc00 con 0x7f2098096500 2024-09-21T09:41:35.738 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.733+0000 7f20977fe640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2084003670 0x7f209812e210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:35.739 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.733+0000 7f2096ffd640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2098075330 0x7f209812e750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:35.739 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.733+0000 7f2097fff640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2098096500 0x7f2098093f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:35.739 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.733+0000 7f20977fe640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2084003670 0x7f209812e210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:43354/0 (socket says 172.21.15.92:43354) 2024-09-21T09:41:35.739 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.733+0000 7f2096ffd640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2098075330 0x7f209812e750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:45008/0 (socket says 172.21.15.92:45008) 2024-09-21T09:41:35.739 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.733+0000 7f20977fe640 1 -- 172.21.15.92:0/315823045 learned_addr learned my addr 172.21.15.92:0/315823045 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:35.739 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.734+0000 7f20977fe640 1 -- 172.21.15.92:0/315823045 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2098096500 msgr2=0x7f2098093f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.739 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.734+0000 7f20977fe640 1 --2- 172.21.15.92:0/315823045 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2098096500 0x7f2098093f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.739 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.734+0000 7f20977fe640 1 -- 172.21.15.92:0/315823045 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2098075330 msgr2=0x7f209812e750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.739 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.734+0000 7f20977fe640 1 --2- 172.21.15.92:0/315823045 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2098075330 0x7f209812e750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.739 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.734+0000 7f20977fe640 1 -- 172.21.15.92:0/315823045 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f209806fde0 con 0x7f2084003670 2024-09-21T09:41:35.739 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.734+0000 7f2096ffd640 1 --2- 172.21.15.92:0/315823045 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2098075330 0x7f209812e750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:35.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.734+0000 7f2097fff640 1 --2- 172.21.15.92:0/315823045 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2098096500 0x7f2098093f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:35.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.734+0000 7f20977fe640 1 --2- 172.21.15.92:0/315823045 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2084003670 0x7f209812e210 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f2088014460 tx=0x7f2088014920 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:35.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.734+0000 7f207ffff640 1 -- 172.21.15.92:0/315823045 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2088012770 con 0x7f2084003670 2024-09-21T09:41:35.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.734+0000 7f207ffff640 1 -- 172.21.15.92:0/315823045 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2088012910 con 0x7f2084003670 2024-09-21T09:41:35.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.734+0000 7f209e12c640 1 -- 172.21.15.92:0/315823045 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f209806f5e0 con 0x7f2084003670 2024-09-21T09:41:35.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.734+0000 7f207ffff640 1 -- 172.21.15.92:0/315823045 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2088012cc0 con 0x7f2084003670 2024-09-21T09:41:35.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.734+0000 7f209e12c640 1 -- 172.21.15.92:0/315823045 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2098094680 con 0x7f2084003670 2024-09-21T09:41:35.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.734+0000 7f207ffff640 1 -- 172.21.15.92:0/315823045 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f208801c030 con 0x7f2084003670 2024-09-21T09:41:35.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.734+0000 7f207ffff640 1 -- 172.21.15.92:0/315823045 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f208801d700 con 0x7f2084003670 2024-09-21T09:41:35.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.735+0000 7f209e12c640 1 -- 172.21.15.92:0/315823045 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2098095ad0 con 0x7f2084003670 2024-09-21T09:41:35.741 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.737+0000 7f207ffff640 1 -- 172.21.15.92:0/315823045 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7f209806fde0 con 0x7f2084003670 2024-09-21T09:41:35.798 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.796+0000 7f209e12c640 1 -- 172.21.15.92:0/315823045 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command([{prefix=config set, name=debug_mds_balancer}] v 0) -- 0x7f20980968e0 con 0x7f2084003670 2024-09-21T09:41:35.803 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.800+0000 7f207ffff640 1 -- 172.21.15.92:0/315823045 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{prefix=config set, name=debug_mds_balancer}]=0 v11)=0 v11) ==== 117+0+0 (secure 0 0 0) 0x7f208801d950 con 0x7f2084003670 2024-09-21T09:41:35.804 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.801+0000 7f209e12c640 1 -- 172.21.15.92:0/315823045 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2084003670 msgr2=0x7f209812e210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.804 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.801+0000 7f209e12c640 1 --2- 172.21.15.92:0/315823045 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2084003670 0x7f209812e210 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f2088014460 tx=0x7f2088014920 comp rx=0 tx=0).stop 2024-09-21T09:41:35.804 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.801+0000 7f209e12c640 1 -- 172.21.15.92:0/315823045 shutdown_connections 2024-09-21T09:41:35.804 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.801+0000 7f209e12c640 1 --2- 172.21.15.92:0/315823045 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2098096500 0x7f2098093f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.805 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.801+0000 7f209e12c640 1 --2- 172.21.15.92:0/315823045 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2098075330 0x7f209812e750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.805 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.801+0000 7f209e12c640 1 --2- 172.21.15.92:0/315823045 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2084003670 0x7f209812e210 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.805 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.801+0000 7f209e12c640 1 -- 172.21.15.92:0/315823045 >> 172.21.15.92:0/315823045 conn(0x7f209809cc90 msgr2=0x7f209808d010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:35.805 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.801+0000 7f209e12c640 1 -- 172.21.15.92:0/315823045 shutdown_connections 2024-09-21T09:41:35.805 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.801+0000 7f209e12c640 1 -- 172.21.15.92:0/315823045 wait complete. 2024-09-21T09:41:35.815 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'debug_ms', '2'], check_status=True, hostname='smithi092') 2024-09-21T09:41:35.815 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'mds_debug_frag', 'True'], check_status=True, hostname='smithi092') 2024-09-21T09:41:35.818 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.815+0000 7f04c1318640 1 -- 172.21.15.92:0/3816009358 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f04bc1302c0 msgr2=0x7f04a800bce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.818 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.815+0000 7f04c1318640 1 --2- 172.21.15.92:0/3816009358 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f04bc1302c0 0x7f04a800bce0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f04b0005ce0 tx=0x7f04b002bc50 comp rx=0 tx=0).stop 2024-09-21T09:41:35.818 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.815+0000 7f04c1318640 1 -- 172.21.15.92:0/3816009358 shutdown_connections 2024-09-21T09:41:35.819 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.815+0000 7f04c1318640 1 --2- 172.21.15.92:0/3816009358 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f04a80036a0 0x7f04a800c220 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.819 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.815+0000 7f04c1318640 1 --2- 172.21.15.92:0/3816009358 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f04bc1302c0 0x7f04a800bce0 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.819 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.815+0000 7f04c1318640 1 --2- 172.21.15.92:0/3816009358 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f04bc12acd0 0x7f04bc12fd80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.819 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.815+0000 7f04c1318640 1 -- 172.21.15.92:0/3816009358 >> 172.21.15.92:0/3816009358 conn(0x7f04bc09d5f0 msgr2=0x7f04bc09da00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:35.819 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.816+0000 7f04c1318640 1 -- 172.21.15.92:0/3816009358 shutdown_connections 2024-09-21T09:41:35.819 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.816+0000 7f04c1318640 1 -- 172.21.15.92:0/3816009358 wait complete. 2024-09-21T09:41:35.821 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.817+0000 7f04c1318640 1 Processor -- start 2024-09-21T09:41:35.822 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.817+0000 7f04c1318640 1 -- start start 2024-09-21T09:41:35.822 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.817+0000 7f04c1318640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f04a80036a0 0x7f04bc06bbc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:35.822 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.817+0000 7f04c1318640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f04bc12acd0 0x7f04bc06c100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:35.822 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.817+0000 7f04c1318640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f04bc12b6c0 0x7f04bc06c640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:35.822 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.817+0000 7f04c1318640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f04bc127a50 con 0x7f04bc12acd0 2024-09-21T09:41:35.822 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.817+0000 7f04c1318640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f04bc1278d0 con 0x7f04a80036a0 2024-09-21T09:41:35.822 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.817+0000 7f04c1318640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f04bc127bd0 con 0x7f04bc12b6c0 2024-09-21T09:41:35.822 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.817+0000 7f04ba7fc640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f04bc12acd0 0x7f04bc06c100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:35.822 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.817+0000 7f04bb7fe640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f04bc12b6c0 0x7f04bc06c640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:35.823 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.817+0000 7f04ba7fc640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f04bc12acd0 0x7f04bc06c100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:43376/0 (socket says 172.21.15.92:43376) 2024-09-21T09:41:35.823 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.817+0000 7f04ba7fc640 1 -- 172.21.15.92:0/3155891503 learned_addr learned my addr 172.21.15.92:0/3155891503 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:35.823 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.817+0000 7f04baffd640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f04a80036a0 0x7f04bc06bbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:35.823 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.817+0000 7f04bb7fe640 1 -- 172.21.15.92:0/3155891503 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f04a80036a0 msgr2=0x7f04bc06bbc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.823 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.817+0000 7f04bb7fe640 1 --2- 172.21.15.92:0/3155891503 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f04a80036a0 0x7f04bc06bbc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.823 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.817+0000 7f04bb7fe640 1 -- 172.21.15.92:0/3155891503 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f04bc12acd0 msgr2=0x7f04bc06c100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.823 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.817+0000 7f04bb7fe640 1 --2- 172.21.15.92:0/3155891503 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f04bc12acd0 0x7f04bc06c100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.824 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.817+0000 7f04bb7fe640 1 -- 172.21.15.92:0/3155891503 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f04bc073180 con 0x7f04bc12b6c0 2024-09-21T09:41:35.824 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.818+0000 7f04bb7fe640 1 --2- 172.21.15.92:0/3155891503 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f04bc12b6c0 0x7f04bc06c640 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f04b40049b0 tx=0x7f04b4013cb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:35.824 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.818+0000 7f04a37fe640 1 -- 172.21.15.92:0/3155891503 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f04b4011b50 con 0x7f04bc12b6c0 2024-09-21T09:41:35.824 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.818+0000 7f04a37fe640 1 -- 172.21.15.92:0/3155891503 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f04b4011d10 con 0x7f04bc12b6c0 2024-09-21T09:41:35.824 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.818+0000 7f04c1318640 1 -- 172.21.15.92:0/3155891503 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f04bc12cb80 con 0x7f04bc12b6c0 2024-09-21T09:41:35.824 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.818+0000 7f04a37fe640 1 -- 172.21.15.92:0/3155891503 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f04b4004d80 con 0x7f04bc12b6c0 2024-09-21T09:41:35.824 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.818+0000 7f04c1318640 1 -- 172.21.15.92:0/3155891503 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f04bc1c3490 con 0x7f04bc12b6c0 2024-09-21T09:41:35.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.818+0000 7f04a37fe640 1 -- 172.21.15.92:0/3155891503 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f04b400c040 con 0x7f04bc12b6c0 2024-09-21T09:41:35.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.818+0000 7f04a37fe640 1 -- 172.21.15.92:0/3155891503 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f04b40255f0 con 0x7f04bc12b6c0 2024-09-21T09:41:35.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.819+0000 7f04c1318640 1 -- 172.21.15.92:0/3155891503 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f04bc128160 con 0x7f04bc12b6c0 2024-09-21T09:41:35.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.821+0000 7f04a37fe640 1 -- 172.21.15.92:0/3155891503 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7f04b401c460 con 0x7f04bc12b6c0 2024-09-21T09:41:35.833 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:35.830+0000 7fdc77016040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:41:35.833 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:35.831+0000 7ffbab4a0040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:41:35.862 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.859+0000 7fc4e72cb640 1 -- 172.21.15.92:0/2484032724 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc4e01292f0 msgr2=0x7fc4c400dff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.862 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.859+0000 7fc4e72cb640 1 --2- 172.21.15.92:0/2484032724 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc4e01292f0 0x7fc4c400dff0 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7fc4d8001cd0 tx=0x7fc4d802a820 comp rx=0 tx=0).stop 2024-09-21T09:41:35.862 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.859+0000 7fc4e72cb640 1 -- 172.21.15.92:0/2484032724 shutdown_connections 2024-09-21T09:41:35.862 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.859+0000 7fc4e72cb640 1 --2- 172.21.15.92:0/2484032724 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc4e01292f0 0x7fc4c400dff0 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.862 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.859+0000 7fc4e72cb640 1 --2- 172.21.15.92:0/2484032724 >> v2:172.21.15.92:3300/0 conn(0x7fc4e01289d0 0x7fc4c40082b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.862 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.859+0000 7fc4e72cb640 1 --2- 172.21.15.92:0/2484032724 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc4c4003670 0x7fc4c4005ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.863 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.859+0000 7fc4e72cb640 1 -- 172.21.15.92:0/2484032724 >> 172.21.15.92:0/2484032724 conn(0x7fc4e009d2b0 msgr2=0x7fc4e009d6c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:35.863 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.859+0000 7fc4e72cb640 1 -- 172.21.15.92:0/2484032724 shutdown_connections 2024-09-21T09:41:35.863 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.859+0000 7fc4e72cb640 1 -- 172.21.15.92:0/2484032724 wait complete. 2024-09-21T09:41:35.863 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.860+0000 7fc4e72cb640 1 Processor -- start 2024-09-21T09:41:35.863 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.860+0000 7fc4e72cb640 1 -- start start 2024-09-21T09:41:35.865 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.861+0000 7fc4e72cb640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc4c4003670 0x7fc4e0122400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:35.865 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.861+0000 7fc4e72cb640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc4e006c100 0x7fc4e0122940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:35.866 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.861+0000 7fc4e72cb640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc4e01289d0 0x7fc4e01c16f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:35.866 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.861+0000 7fc4e72cb640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fc4e006c550 con 0x7fc4e01289d0 2024-09-21T09:41:35.866 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.861+0000 7fc4e72cb640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fc4e0129040 con 0x7fc4e006c100 2024-09-21T09:41:35.866 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.861+0000 7fc4e4866640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc4e006c100 0x7fc4e0122940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:35.866 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.861+0000 7fc4e4866640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc4e006c100 0x7fc4e0122940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:45038/0 (socket says 172.21.15.92:45038) 2024-09-21T09:41:35.866 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.861+0000 7fc4e4866640 1 -- 172.21.15.92:0/965009376 learned_addr learned my addr 172.21.15.92:0/965009376 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:35.866 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.861+0000 7fc4e5067640 1 --2- 172.21.15.92:0/965009376 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc4c4003670 0x7fc4e0122400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:35.866 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.861+0000 7fc4e5868640 1 --2- 172.21.15.92:0/965009376 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc4e01289d0 0x7fc4e01c16f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:35.866 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.861+0000 7fc4e72cb640 1 -- 172.21.15.92:0/965009376 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fc4e006c6d0 con 0x7fc4c4003670 2024-09-21T09:41:35.866 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.861+0000 7fc4e5868640 1 -- 172.21.15.92:0/965009376 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc4c4003670 msgr2=0x7fc4e0122400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.866 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.861+0000 7fc4e5868640 1 --2- 172.21.15.92:0/965009376 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc4c4003670 0x7fc4e0122400 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.866 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.862+0000 7fc4e5868640 1 -- 172.21.15.92:0/965009376 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc4e006c100 msgr2=0x7fc4e0122940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.867 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.862+0000 7fc4e5868640 1 --2- 172.21.15.92:0/965009376 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc4e006c100 0x7fc4e0122940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.867 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.862+0000 7fc4e5868640 1 -- 172.21.15.92:0/965009376 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc4e0125910 con 0x7fc4e01289d0 2024-09-21T09:41:35.867 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.862+0000 7fc4e4866640 1 --2- 172.21.15.92:0/965009376 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc4e006c100 0x7fc4e0122940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:35.867 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.862+0000 7fc4e5868640 1 --2- 172.21.15.92:0/965009376 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc4e01289d0 0x7fc4e01c16f0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7fc4d8004000 tx=0x7fc4d8003ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:35.867 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.862+0000 7fc4d5ffb640 1 -- 172.21.15.92:0/965009376 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc4d8041020 con 0x7fc4e01289d0 2024-09-21T09:41:35.867 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.862+0000 7fc4d5ffb640 1 -- 172.21.15.92:0/965009376 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fc4d803d700 con 0x7fc4e01289d0 2024-09-21T09:41:35.867 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.862+0000 7fc4d5ffb640 1 -- 172.21.15.92:0/965009376 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc4d8038040 con 0x7fc4e01289d0 2024-09-21T09:41:35.867 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.862+0000 7fc4e72cb640 1 -- 172.21.15.92:0/965009376 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc4e0124520 con 0x7fc4e01289d0 2024-09-21T09:41:35.867 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.862+0000 7fc4e72cb640 1 -- 172.21.15.92:0/965009376 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc4e01c1c30 con 0x7fc4e01289d0 2024-09-21T09:41:35.867 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.863+0000 7fc4d5ffb640 1 -- 172.21.15.92:0/965009376 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7fc4d8044020 con 0x7fc4e01289d0 2024-09-21T09:41:35.867 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.863+0000 7fc4d5ffb640 1 -- 172.21.15.92:0/965009376 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7fc4d80333f0 con 0x7fc4e01289d0 2024-09-21T09:41:35.867 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.863+0000 7fc4e72cb640 1 -- 172.21.15.92:0/965009376 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc4e01234f0 con 0x7fc4e01289d0 2024-09-21T09:41:35.870 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.867+0000 7fc4d5ffb640 1 -- 172.21.15.92:0/965009376 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7fc4d8046410 con 0x7fc4e01289d0 2024-09-21T09:41:35.884 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.882+0000 7f04c1318640 1 -- 172.21.15.92:0/3155891503 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command([{prefix=config set, name=mds_debug_frag}] v 0) -- 0x7f04bc068d20 con 0x7f04bc12b6c0 2024-09-21T09:41:35.890 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.888+0000 7f04a37fe640 1 -- 172.21.15.92:0/3155891503 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mon_command_ack([{prefix=config set, name=mds_debug_frag}]=0 v12)=0 v12) ==== 115+0+0 (secure 0 0 0) 0x7f04b40258a0 con 0x7f04bc12b6c0 2024-09-21T09:41:35.892 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.889+0000 7f04c1318640 1 -- 172.21.15.92:0/3155891503 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f04bc12b6c0 msgr2=0x7f04bc06c640 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.892 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.889+0000 7f04c1318640 1 --2- 172.21.15.92:0/3155891503 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f04bc12b6c0 0x7f04bc06c640 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f04b40049b0 tx=0x7f04b4013cb0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.892 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.889+0000 7f04c1318640 1 -- 172.21.15.92:0/3155891503 shutdown_connections 2024-09-21T09:41:35.892 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.889+0000 7f04c1318640 1 --2- 172.21.15.92:0/3155891503 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f04bc12b6c0 0x7f04bc06c640 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.892 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.889+0000 7f04c1318640 1 --2- 172.21.15.92:0/3155891503 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f04bc12acd0 0x7f04bc06c100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.892 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.889+0000 7f04c1318640 1 --2- 172.21.15.92:0/3155891503 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f04a80036a0 0x7f04bc06bbc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.892 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.889+0000 7f04c1318640 1 -- 172.21.15.92:0/3155891503 >> 172.21.15.92:0/3155891503 conn(0x7f04bc09d5f0 msgr2=0x7f04bc125f30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:35.892 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.889+0000 7f04c1318640 1 -- 172.21.15.92:0/3155891503 shutdown_connections 2024-09-21T09:41:35.892 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.889+0000 7f04c1318640 1 -- 172.21.15.92:0/3155891503 wait complete. 2024-09-21T09:41:35.902 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'mds_debug_scatterstat', 'True'], check_status=True, hostname='smithi092') 2024-09-21T09:41:35.907 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.904+0000 7f885919c640 1 -- 172.21.15.92:0/2929480176 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f8854133b90 msgr2=0x7f884000add0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.907 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.904+0000 7f885919c640 1 --2- 172.21.15.92:0/2929480176 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f8854133b90 0x7f884000add0 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f8854075640 tx=0x7f884c02bbb0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.907 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.904+0000 7f885919c640 1 -- 172.21.15.92:0/2929480176 shutdown_connections 2024-09-21T09:41:35.907 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.904+0000 7f885919c640 1 --2- 172.21.15.92:0/2929480176 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f8854133b90 0x7f884000add0 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.907 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.904+0000 7f885919c640 1 --2- 172.21.15.92:0/2929480176 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f885409ac00 0x7f8840008860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.907 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.904+0000 7f885919c640 1 --2- 172.21.15.92:0/2929480176 >> v2:172.21.15.92:3300/0 conn(0x7f8840003b80 0x7f8840006310 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.907 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.904+0000 7f885919c640 1 -- 172.21.15.92:0/2929480176 >> 172.21.15.92:0/2929480176 conn(0x7f885409e830 msgr2=0x7f88540a0300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:35.908 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.904+0000 7f885919c640 1 -- 172.21.15.92:0/2929480176 shutdown_connections 2024-09-21T09:41:35.908 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.904+0000 7f885919c640 1 -- 172.21.15.92:0/2929480176 wait complete. 2024-09-21T09:41:35.910 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.905+0000 7f885919c640 1 Processor -- start 2024-09-21T09:41:35.910 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.905+0000 7f885919c640 1 -- start start 2024-09-21T09:41:35.910 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.906+0000 7f885919c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f8840003b80 0x7f88541c3fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:35.910 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.906+0000 7f885919c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f885409ac00 0x7f88541c44e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:35.910 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.906+0000 7f885919c640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f885409b520 0x7f885408caa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:35.910 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.906+0000 7f885919c640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f885412e5d0 con 0x7f8840003b80 2024-09-21T09:41:35.910 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.906+0000 7f885359e640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f885409b520 0x7f885408caa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:35.910 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.906+0000 7f885359e640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f885409b520 0x7f885408caa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:54072/0 (socket says 172.21.15.92:54072) 2024-09-21T09:41:35.911 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.906+0000 7f885359e640 1 -- 172.21.15.92:0/4115357802 learned_addr learned my addr 172.21.15.92:0/4115357802 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:35.911 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.906+0000 7f885259c640 1 --2- 172.21.15.92:0/4115357802 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f885409ac00 0x7f88541c44e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:35.911 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.906+0000 7f885919c640 1 -- 172.21.15.92:0/4115357802 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f885412e450 con 0x7f885409ac00 2024-09-21T09:41:35.911 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.906+0000 7f885919c640 1 -- 172.21.15.92:0/4115357802 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f885412e750 con 0x7f885409b520 2024-09-21T09:41:35.911 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.906+0000 7f8852d9d640 1 --2- 172.21.15.92:0/4115357802 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f8840003b80 0x7f88541c3fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:35.911 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.906+0000 7f885359e640 1 -- 172.21.15.92:0/4115357802 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f885409ac00 msgr2=0x7f88541c44e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.911 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.906+0000 7f885359e640 1 --2- 172.21.15.92:0/4115357802 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f885409ac00 0x7f88541c44e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.911 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.906+0000 7f885359e640 1 -- 172.21.15.92:0/4115357802 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f8840003b80 msgr2=0x7f88541c3fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.911 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.906+0000 7f885359e640 1 --2- 172.21.15.92:0/4115357802 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f8840003b80 0x7f88541c3fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.911 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.906+0000 7f885359e640 1 -- 172.21.15.92:0/4115357802 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f885408e610 con 0x7f885409b520 2024-09-21T09:41:35.911 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.907+0000 7f885359e640 1 --2- 172.21.15.92:0/4115357802 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f885409b520 0x7f885408caa0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f884c004500 tx=0x7f884c009310 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:35.911 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.907+0000 7f883b7fe640 1 -- 172.21.15.92:0/4115357802 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f884c037600 con 0x7f885409b520 2024-09-21T09:41:35.911 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.907+0000 7f883b7fe640 1 -- 172.21.15.92:0/4115357802 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f884c046070 con 0x7f885409b520 2024-09-21T09:41:35.911 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.907+0000 7f885919c640 1 -- 172.21.15.92:0/4115357802 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8854099ae0 con 0x7f885409b520 2024-09-21T09:41:35.912 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.907+0000 7f883b7fe640 1 -- 172.21.15.92:0/4115357802 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f884c037b30 con 0x7f885409b520 2024-09-21T09:41:35.912 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.907+0000 7f885919c640 1 -- 172.21.15.92:0/4115357802 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f885408f6f0 con 0x7f885409b520 2024-09-21T09:41:35.912 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.907+0000 7f883b7fe640 1 -- 172.21.15.92:0/4115357802 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f884c00c040 con 0x7f885409b520 2024-09-21T09:41:35.912 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.907+0000 7f883b7fe640 1 -- 172.21.15.92:0/4115357802 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f884c04bdc0 con 0x7f885409b520 2024-09-21T09:41:35.912 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.908+0000 7f885919c640 1 -- 172.21.15.92:0/4115357802 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f88540008d0 con 0x7f885409b520 2024-09-21T09:41:35.915 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.912+0000 7f883b7fe640 1 -- 172.21.15.92:0/4115357802 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7f884c037cd0 con 0x7f885409b520 2024-09-21T09:41:35.951 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.948+0000 7f61fa51f640 1 -- 172.21.15.92:0/3626491247 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f61f412edd0 msgr2=0x7f61e000bb90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.951 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.948+0000 7f61fa51f640 1 --2- 172.21.15.92:0/3626491247 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f61f412edd0 0x7f61e000bb90 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f61f4075340 tx=0x7f61ec02bca0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.951 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.948+0000 7f61fa51f640 1 -- 172.21.15.92:0/3626491247 shutdown_connections 2024-09-21T09:41:35.951 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.948+0000 7f61fa51f640 1 --2- 172.21.15.92:0/3626491247 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f61f412edd0 0x7f61e000bb90 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.951 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.948+0000 7f61fa51f640 1 --2- 172.21.15.92:0/3626491247 >> v2:172.21.15.92:3300/0 conn(0x7f61e00036d0 0x7f61e0005b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.951 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.948+0000 7f61fa51f640 1 --2- 172.21.15.92:0/3626491247 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61f4131700 0x7f61f4133af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.951 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.948+0000 7f61fa51f640 1 -- 172.21.15.92:0/3626491247 >> 172.21.15.92:0/3626491247 conn(0x7f61f409e4f0 msgr2=0x7f61f409e900 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:35.951 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.948+0000 7f61fa51f640 1 -- 172.21.15.92:0/3626491247 shutdown_connections 2024-09-21T09:41:35.952 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.948+0000 7f61fa51f640 1 -- 172.21.15.92:0/3626491247 wait complete. 2024-09-21T09:41:35.952 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.949+0000 7f61fa51f640 1 Processor -- start 2024-09-21T09:41:35.952 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.950+0000 7f61fa51f640 1 -- start start 2024-09-21T09:41:35.953 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.950+0000 7f61fa51f640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f61e00036d0 0x7f61f4095810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:35.953 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.950+0000 7f61fa51f640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61f40925f0 0x7f61f4095d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:35.953 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.950+0000 7f61f37fe640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61f40925f0 0x7f61f4095d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:35.953 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.950+0000 7f61f37fe640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61f40925f0 0x7f61f4095d50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:43450/0 (socket says 172.21.15.92:43450) 2024-09-21T09:41:35.953 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.950+0000 7f61f3fff640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f61e00036d0 0x7f61f4095810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:35.953 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.950+0000 7f61f3fff640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f61e00036d0 0x7f61f4095810 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:45080/0 (socket says 172.21.15.92:45080) 2024-09-21T09:41:35.954 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.950+0000 7f61fa51f640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61f412edd0 0x7f61f4096290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:35.954 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.950+0000 7fc4e72cb640 1 -- 172.21.15.92:0/965009376 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command([{prefix=config set, name=mds_debug_scatterstat}] v 0) -- 0x7fc4e012ced0 con 0x7fc4e01289d0 2024-09-21T09:41:35.956 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.950+0000 7f61f37fe640 1 -- 172.21.15.92:0/3946356764 learned_addr learned my addr 172.21.15.92:0/3946356764 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:35.956 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.950+0000 7f61fa51f640 1 -- 172.21.15.92:0/3946356764 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f61f412bc40 con 0x7f61f40925f0 2024-09-21T09:41:35.956 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.951+0000 7f61fa51f640 1 -- 172.21.15.92:0/3946356764 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f61f412bac0 con 0x7f61e00036d0 2024-09-21T09:41:35.956 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.951+0000 7f61fa51f640 1 -- 172.21.15.92:0/3946356764 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f61f412bdc0 con 0x7f61f412edd0 2024-09-21T09:41:35.956 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.951+0000 7f61f8abc640 1 --2- 172.21.15.92:0/3946356764 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61f412edd0 0x7f61f4096290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:35.956 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.951+0000 7f61f37fe640 1 -- 172.21.15.92:0/3946356764 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61f412edd0 msgr2=0x7f61f4096290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.956 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.951+0000 7f61f37fe640 1 --2- 172.21.15.92:0/3946356764 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61f412edd0 0x7f61f4096290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.956 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.951+0000 7f61f37fe640 1 -- 172.21.15.92:0/3946356764 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f61e00036d0 msgr2=0x7f61f4095810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.956 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.951+0000 7f61f37fe640 1 --2- 172.21.15.92:0/3946356764 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f61e00036d0 0x7f61f4095810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.956 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.951+0000 7f61f37fe640 1 -- 172.21.15.92:0/3946356764 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f61f4128960 con 0x7f61f40925f0 2024-09-21T09:41:35.956 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.951+0000 7f61f37fe640 1 --2- 172.21.15.92:0/3946356764 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61f40925f0 0x7f61f4095d50 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f61e8013830 tx=0x7f61e8013cf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:35.956 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.951+0000 7f61f0ff9640 1 -- 172.21.15.92:0/3946356764 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f61e8004440 con 0x7f61f40925f0 2024-09-21T09:41:35.957 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.952+0000 7f61f0ff9640 1 -- 172.21.15.92:0/3946356764 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f61e800f940 con 0x7f61f40925f0 2024-09-21T09:41:35.957 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.952+0000 7f61f0ff9640 1 -- 172.21.15.92:0/3946356764 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f61e800fd90 con 0x7f61f40925f0 2024-09-21T09:41:35.957 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.952+0000 7f61fa51f640 1 -- 172.21.15.92:0/3946356764 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f61f4098220 con 0x7f61f40925f0 2024-09-21T09:41:35.957 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.952+0000 7f61fa51f640 1 -- 172.21.15.92:0/3946356764 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f61f41c18a0 con 0x7f61f40925f0 2024-09-21T09:41:35.957 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.952+0000 7f61f0ff9640 1 -- 172.21.15.92:0/3946356764 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f61e80035f0 con 0x7f61f40925f0 2024-09-21T09:41:35.957 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.952+0000 7f61f0ff9640 1 -- 172.21.15.92:0/3946356764 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f61e80253f0 con 0x7f61f40925f0 2024-09-21T09:41:35.957 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:35.952+0000 7fdc77016040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:41:35.957 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.953+0000 7f61fa51f640 1 -- 172.21.15.92:0/3946356764 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f61f409e970 con 0x7f61f40925f0 2024-09-21T09:41:35.958 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:35.954+0000 7ffbab4a0040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:41:35.958 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.955+0000 7f61f0ff9640 1 -- 172.21.15.92:0/3946356764 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7f61f409e970 con 0x7f61f40925f0 2024-09-21T09:41:35.959 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.957+0000 7fc4d5ffb640 1 -- 172.21.15.92:0/965009376 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mds_debug_scatterstat}]=0 v13)=0 v13) ==== 122+0+0 (secure 0 0 0) 0x7fc4d80465f0 con 0x7fc4e01289d0 2024-09-21T09:41:35.961 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.958+0000 7fc4e72cb640 1 -- 172.21.15.92:0/965009376 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc4e01289d0 msgr2=0x7fc4e01c16f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.961 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.958+0000 7fc4e72cb640 1 --2- 172.21.15.92:0/965009376 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc4e01289d0 0x7fc4e01c16f0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7fc4d8004000 tx=0x7fc4d8003ba0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.961 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.958+0000 7fc4e72cb640 1 -- 172.21.15.92:0/965009376 shutdown_connections 2024-09-21T09:41:35.961 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.958+0000 7fc4e72cb640 1 --2- 172.21.15.92:0/965009376 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc4e01289d0 0x7fc4e01c16f0 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.961 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.958+0000 7fc4e72cb640 1 --2- 172.21.15.92:0/965009376 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc4e006c100 0x7fc4e0122940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.961 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.958+0000 7fc4e72cb640 1 --2- 172.21.15.92:0/965009376 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc4c4003670 0x7fc4e0122400 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.962 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.958+0000 7fc4e72cb640 1 -- 172.21.15.92:0/965009376 >> 172.21.15.92:0/965009376 conn(0x7fc4e009d2b0 msgr2=0x7fc4e01b1df0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:35.962 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.958+0000 7fc4e72cb640 1 -- 172.21.15.92:0/965009376 shutdown_connections 2024-09-21T09:41:35.962 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.958+0000 7fc4e72cb640 1 -- 172.21.15.92:0/965009376 wait complete. 2024-09-21T09:41:35.976 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'mds_op_complaint_time', '180'], check_status=True, hostname='smithi092') 2024-09-21T09:41:35.993 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.990+0000 7f612d8eb640 1 -- 172.21.15.92:0/527487776 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f612812c400 msgr2=0x7f61080083d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.994 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.990+0000 7f612d8eb640 1 --2- 172.21.15.92:0/527487776 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f612812c400 0x7f61080083d0 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f6110005ce0 tx=0x7f611002bbd0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.994 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.990+0000 7f612d8eb640 1 -- 172.21.15.92:0/527487776 shutdown_connections 2024-09-21T09:41:35.994 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.990+0000 7f612d8eb640 1 --2- 172.21.15.92:0/527487776 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f612812b9b0 0x7f610800a980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.994 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.990+0000 7f612d8eb640 1 --2- 172.21.15.92:0/527487776 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f612812c400 0x7f61080083d0 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.994 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.990+0000 7f612d8eb640 1 --2- 172.21.15.92:0/527487776 >> v2:172.21.15.92:3300/0 conn(0x7f6108003700 0x7f6108005b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.994 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.990+0000 7f612d8eb640 1 -- 172.21.15.92:0/527487776 >> 172.21.15.92:0/527487776 conn(0x7f612809e280 msgr2=0x7f612809e660 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:35.994 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.990+0000 7f612d8eb640 1 -- 172.21.15.92:0/527487776 shutdown_connections 2024-09-21T09:41:35.994 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.990+0000 7f612d8eb640 1 -- 172.21.15.92:0/527487776 wait complete. 2024-09-21T09:41:35.994 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.992+0000 7f612d8eb640 1 Processor -- start 2024-09-21T09:41:35.995 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.992+0000 7f612d8eb640 1 -- start start 2024-09-21T09:41:35.996 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.993+0000 7f612d8eb640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6108003700 0x7f6128124770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:35.997 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.993+0000 7f6126ffd640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6108003700 0x7f6128124770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:35.997 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.993+0000 7f6126ffd640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6108003700 0x7f6128124770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:45096/0 (socket says 172.21.15.92:45096) 2024-09-21T09:41:35.997 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.993+0000 7f612d8eb640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61281289b0 0x7f6128124cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:35.997 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.994+0000 7f6126ffd640 1 -- 172.21.15.92:0/2443779737 learned_addr learned my addr 172.21.15.92:0/2443779737 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:35.997 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.994+0000 7f612d8eb640 1 --2- 172.21.15.92:0/2443779737 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f612812b9b0 0x7f61281251f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:35.997 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.994+0000 7f612d8eb640 1 -- 172.21.15.92:0/2443779737 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f6128128fc0 con 0x7f612812b9b0 2024-09-21T09:41:35.997 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.994+0000 7f61267fc640 1 --2- 172.21.15.92:0/2443779737 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61281289b0 0x7f6128124cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:35.997 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.994+0000 7f61277fe640 1 --2- 172.21.15.92:0/2443779737 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f612812b9b0 0x7f61281251f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:35.997 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.994+0000 7f612d8eb640 1 -- 172.21.15.92:0/2443779737 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f6128128e40 con 0x7f6108003700 2024-09-21T09:41:35.997 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.994+0000 7f612d8eb640 1 -- 172.21.15.92:0/2443779737 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f6128129140 con 0x7f61281289b0 2024-09-21T09:41:35.999 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.994+0000 7f61277fe640 1 -- 172.21.15.92:0/2443779737 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61281289b0 msgr2=0x7f6128124cb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.999 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.994+0000 7f61277fe640 1 --2- 172.21.15.92:0/2443779737 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61281289b0 0x7f6128124cb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.999 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.994+0000 7f61277fe640 1 -- 172.21.15.92:0/2443779737 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6108003700 msgr2=0x7f6128124770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:35.999 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.994+0000 7f61277fe640 1 --2- 172.21.15.92:0/2443779737 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6108003700 0x7f6128124770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:35.999 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.994+0000 7f61277fe640 1 -- 172.21.15.92:0/2443779737 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6128127330 con 0x7f612812b9b0 2024-09-21T09:41:35.999 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.994+0000 7f61267fc640 1 --2- 172.21.15.92:0/2443779737 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61281289b0 0x7f6128124cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:36.001 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.995+0000 7f61277fe640 1 --2- 172.21.15.92:0/2443779737 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f612812b9b0 0x7f61281251f0 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f61200062e0 tx=0x7f61200066e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:36.001 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.995+0000 7f611ffff640 1 -- 172.21.15.92:0/2443779737 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f612001e070 con 0x7f612812b9b0 2024-09-21T09:41:36.001 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.995+0000 7f611ffff640 1 -- 172.21.15.92:0/2443779737 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f6120001bc0 con 0x7f612812b9b0 2024-09-21T09:41:36.002 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.995+0000 7f611ffff640 1 -- 172.21.15.92:0/2443779737 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6120002010 con 0x7f612812b9b0 2024-09-21T09:41:36.002 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.995+0000 7f612d8eb640 1 -- 172.21.15.92:0/2443779737 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6128095230 con 0x7f612812b9b0 2024-09-21T09:41:36.002 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.995+0000 7f612d8eb640 1 -- 172.21.15.92:0/2443779737 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f61281c1600 con 0x7f612812b9b0 2024-09-21T09:41:36.002 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.996+0000 7f611ffff640 1 -- 172.21.15.92:0/2443779737 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f6120021070 con 0x7f612812b9b0 2024-09-21T09:41:36.002 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.996+0000 7f611ffff640 1 -- 172.21.15.92:0/2443779737 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f6120019530 con 0x7f612812b9b0 2024-09-21T09:41:36.002 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:35.996+0000 7f612d8eb640 1 -- 172.21.15.92:0/2443779737 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f612812cf40 con 0x7f612812b9b0 2024-09-21T09:41:36.003 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.000+0000 7f611ffff640 1 -- 172.21.15.92:0/2443779737 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7f6120028070 con 0x7f612812b9b0 2024-09-21T09:41:36.003 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.001+0000 7f885919c640 1 -- 172.21.15.92:0/4115357802 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command([{prefix=config set, name=mds_op_complaint_time}] v 0) -- 0x7f885408f950 con 0x7f885409b520 2024-09-21T09:41:36.009 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.007+0000 7f883b7fe640 1 -- 172.21.15.92:0/4115357802 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mon_command_ack([{prefix=config set, name=mds_op_complaint_time}]=0 v14)=0 v14) ==== 121+0+0 (secure 0 0 0) 0x7f884c04b460 con 0x7f885409b520 2024-09-21T09:41:36.011 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.008+0000 7f885919c640 1 -- 172.21.15.92:0/4115357802 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f885409b520 msgr2=0x7f885408caa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.011 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.008+0000 7f885919c640 1 --2- 172.21.15.92:0/4115357802 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f885409b520 0x7f885408caa0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f884c004500 tx=0x7f884c009310 comp rx=0 tx=0).stop 2024-09-21T09:41:36.011 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.008+0000 7f885919c640 1 -- 172.21.15.92:0/4115357802 shutdown_connections 2024-09-21T09:41:36.011 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.008+0000 7f885919c640 1 --2- 172.21.15.92:0/4115357802 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f885409b520 0x7f885408caa0 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.011 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.008+0000 7f885919c640 1 --2- 172.21.15.92:0/4115357802 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f885409ac00 0x7f88541c44e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.011 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.008+0000 7f885919c640 1 --2- 172.21.15.92:0/4115357802 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f8840003b80 0x7f88541c3fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.011 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.008+0000 7f885919c640 1 -- 172.21.15.92:0/4115357802 >> 172.21.15.92:0/4115357802 conn(0x7f885409e830 msgr2=0x7f8854096b80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:36.011 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.008+0000 7f885919c640 1 -- 172.21.15.92:0/4115357802 shutdown_connections 2024-09-21T09:41:36.011 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.008+0000 7f885919c640 1 -- 172.21.15.92:0/4115357802 wait complete. 2024-09-21T09:41:36.022 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'mds_verify_scatter', 'True'], check_status=True, hostname='smithi092') 2024-09-21T09:41:36.028 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.026+0000 7f61fa51f640 1 -- 172.21.15.92:0/3946356764 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command([{prefix=config set, name=mds_verify_scatter}] v 0) -- 0x7f61f412c2c0 con 0x7f61f40925f0 2024-09-21T09:41:36.033 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.030+0000 7f61f0ff9640 1 -- 172.21.15.92:0/3946356764 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mds_verify_scatter}]=0 v15)=0 v15) ==== 119+0+0 (secure 0 0 0) 0x7f61e801c440 con 0x7f61f40925f0 2024-09-21T09:41:36.034 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.031+0000 7f61fa51f640 1 -- 172.21.15.92:0/3946356764 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61f40925f0 msgr2=0x7f61f4095d50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.034 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.031+0000 7f61fa51f640 1 --2- 172.21.15.92:0/3946356764 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61f40925f0 0x7f61f4095d50 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f61e8013830 tx=0x7f61e8013cf0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.034 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.032+0000 7f61fa51f640 1 -- 172.21.15.92:0/3946356764 shutdown_connections 2024-09-21T09:41:36.036 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.032+0000 7f61fa51f640 1 --2- 172.21.15.92:0/3946356764 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61f412edd0 0x7f61f4096290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.036 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.032+0000 7f61fa51f640 1 --2- 172.21.15.92:0/3946356764 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61f40925f0 0x7f61f4095d50 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.036 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.032+0000 7f61fa51f640 1 --2- 172.21.15.92:0/3946356764 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f61e00036d0 0x7f61f4095810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.036 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.032+0000 7f61fa51f640 1 -- 172.21.15.92:0/3946356764 >> 172.21.15.92:0/3946356764 conn(0x7f61f409e4f0 msgr2=0x7f61f412f1b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:36.036 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.033+0000 7f61fa51f640 1 -- 172.21.15.92:0/3946356764 shutdown_connections 2024-09-21T09:41:36.036 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.033+0000 7f61fa51f640 1 -- 172.21.15.92:0/3946356764 wait complete. 2024-09-21T09:41:36.037 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.034+0000 7f9426965640 1 -- 172.21.15.92:0/966890256 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f9420131810 msgr2=0x7f940c0082d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.037 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.034+0000 7f9426965640 1 --2- 172.21.15.92:0/966890256 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f9420131810 0x7f940c0082d0 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f9410006ba0 tx=0x7f941002b9d0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.037 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.034+0000 7f9426965640 1 -- 172.21.15.92:0/966890256 shutdown_connections 2024-09-21T09:41:36.037 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.034+0000 7f9426965640 1 --2- 172.21.15.92:0/966890256 >> v2:172.21.15.92:3300/0 conn(0x7f942012eee0 0x7f940c00e070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.037 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.034+0000 7f9426965640 1 --2- 172.21.15.92:0/966890256 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f9420131810 0x7f940c0082d0 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.038 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.034+0000 7f9426965640 1 --2- 172.21.15.92:0/966890256 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f940c003670 0x7f940c005ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.038 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.034+0000 7f9426965640 1 -- 172.21.15.92:0/966890256 >> 172.21.15.92:0/966890256 conn(0x7f942009d2b0 msgr2=0x7f942009d6c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:36.038 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.034+0000 7f9426965640 1 -- 172.21.15.92:0/966890256 shutdown_connections 2024-09-21T09:41:36.038 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.034+0000 7f9426965640 1 -- 172.21.15.92:0/966890256 wait complete. 2024-09-21T09:41:36.039 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.035+0000 7f9426965640 1 Processor -- start 2024-09-21T09:41:36.039 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.036+0000 7f9426965640 1 -- start start 2024-09-21T09:41:36.039 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.036+0000 7f9426965640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f940c003670 0x7f9420124d70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:36.039 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.036+0000 7f9426965640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f94201289d0 0x7f94201252b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:36.039 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.036+0000 7f9426965640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f942012eee0 0x7f94201257f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:36.039 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.036+0000 7f9426965640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f942006bbc0 con 0x7f940c003670 2024-09-21T09:41:36.039 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.036+0000 7f9426965640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f942006bd40 con 0x7f942012eee0 2024-09-21T09:41:36.039 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.036+0000 7f9426965640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f942006bec0 con 0x7f94201289d0 2024-09-21T09:41:36.039 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.036+0000 7f941ffff640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f940c003670 0x7f9420124d70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:36.039 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.036+0000 7f941f7fe640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f94201289d0 0x7f94201252b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:36.040 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.036+0000 7f941f7fe640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f94201289d0 0x7f94201252b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:54108/0 (socket says 172.21.15.92:54108) 2024-09-21T09:41:36.040 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.036+0000 7f941f7fe640 1 -- 172.21.15.92:0/3937719757 learned_addr learned my addr 172.21.15.92:0/3937719757 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:36.040 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.036+0000 7f9424f02640 1 --2- 172.21.15.92:0/3937719757 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f942012eee0 0x7f94201257f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:36.041 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.037+0000 7f941f7fe640 1 -- 172.21.15.92:0/3937719757 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f942012eee0 msgr2=0x7f94201257f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.041 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.037+0000 7f941f7fe640 1 --2- 172.21.15.92:0/3937719757 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f942012eee0 0x7f94201257f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.041 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.037+0000 7f941f7fe640 1 -- 172.21.15.92:0/3937719757 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f940c003670 msgr2=0x7f9420124d70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.041 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.037+0000 7f941f7fe640 1 --2- 172.21.15.92:0/3937719757 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f940c003670 0x7f9420124d70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.041 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.037+0000 7f941f7fe640 1 -- 172.21.15.92:0/3937719757 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f942012e000 con 0x7f94201289d0 2024-09-21T09:41:36.041 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.037+0000 7f941f7fe640 1 --2- 172.21.15.92:0/3937719757 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f94201289d0 0x7f94201252b0 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f94140059d0 tx=0x7f9414013c60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:36.041 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.037+0000 7f941cff9640 1 -- 172.21.15.92:0/3937719757 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f941400fdb0 con 0x7f94201289d0 2024-09-21T09:41:36.041 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.037+0000 7f941cff9640 1 -- 172.21.15.92:0/3937719757 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f941401d3f0 con 0x7f94201289d0 2024-09-21T09:41:36.041 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.037+0000 7f941cff9640 1 -- 172.21.15.92:0/3937719757 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f941401d860 con 0x7f94201289d0 2024-09-21T09:41:36.042 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.037+0000 7f9426965640 1 -- 172.21.15.92:0/3937719757 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f94200958d0 con 0x7f94201289d0 2024-09-21T09:41:36.042 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.037+0000 7f9426965640 1 -- 172.21.15.92:0/3937719757 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f9420095ca0 con 0x7f94201289d0 2024-09-21T09:41:36.042 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.038+0000 7f941cff9640 1 -- 172.21.15.92:0/3937719757 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f941401da00 con 0x7f94201289d0 2024-09-21T09:41:36.042 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.038+0000 7f941cff9640 1 -- 172.21.15.92:0/3937719757 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f941401c3a0 con 0x7f94201289d0 2024-09-21T09:41:36.042 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.039+0000 7f9426965640 1 -- 172.21.15.92:0/3937719757 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f942006c450 con 0x7f94201289d0 2024-09-21T09:41:36.045 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.042+0000 7f941cff9640 1 -- 172.21.15.92:0/3937719757 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7f9414018070 con 0x7f94201289d0 2024-09-21T09:41:36.049 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'osd_op_complaint_time', '180'], check_status=True, hostname='smithi092') 2024-09-21T09:41:36.079 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.076+0000 7f77eb7fe640 1 -- 172.21.15.92:0/833569482 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f77d803a400 con 0x7f77d0003670 2024-09-21T09:41:36.080 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.077+0000 7f77f2c55640 1 -- 172.21.15.92:0/833569482 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f77d0003670 msgr2=0x7f77d0009a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.080 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.077+0000 7f77f2c55640 1 --2- 172.21.15.92:0/833569482 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f77d0003670 0x7f77d0009a90 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f77d8005e50 tx=0x7f77d8030030 comp rx=0 tx=0).stop 2024-09-21T09:41:36.080 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.077+0000 7f77f2c55640 1 -- 172.21.15.92:0/833569482 shutdown_connections 2024-09-21T09:41:36.080 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.077+0000 7f77f2c55640 1 --2- 172.21.15.92:0/833569482 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f77ec130500 0x7f77d000bfe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.081 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.077+0000 7f77f2c55640 1 --2- 172.21.15.92:0/833569482 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f77d0003670 0x7f77d0009a90 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.081 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.077+0000 7f77f2c55640 1 --2- 172.21.15.92:0/833569482 >> v2:172.21.15.92:3300/0 conn(0x7f77ec12ad40 0x7f77ec127320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.081 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.077+0000 7f77f2c55640 1 -- 172.21.15.92:0/833569482 >> 172.21.15.92:0/833569482 conn(0x7f77ec09c6c0 msgr2=0x7f77ec09cad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:36.081 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.078+0000 7f77f2c55640 1 -- 172.21.15.92:0/833569482 shutdown_connections 2024-09-21T09:41:36.081 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.078+0000 7f77f2c55640 1 -- 172.21.15.92:0/833569482 wait complete. 2024-09-21T09:41:36.081 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.079+0000 7f77f2c55640 1 Processor -- start 2024-09-21T09:41:36.081 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.079+0000 7f77f2c55640 1 -- start start 2024-09-21T09:41:36.083 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.080+0000 7f77f2c55640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f77d0003670 0x7f77ec12e5a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:36.084 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.080+0000 7f77f2c55640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f77ec12a960 0x7f77ec12eae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:36.084 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.080+0000 7f77f09f1640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f77d0003670 0x7f77ec12e5a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:36.084 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.080+0000 7f77f09f1640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f77d0003670 0x7f77ec12e5a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:43478/0 (socket says 172.21.15.92:43478) 2024-09-21T09:41:36.084 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.080+0000 7f77ebfff640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f77ec12a960 0x7f77ec12eae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:36.084 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.080+0000 7f77ebfff640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f77ec12a960 0x7f77ec12eae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:54112/0 (socket says 172.21.15.92:54112) 2024-09-21T09:41:36.084 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.080+0000 7f77f09f1640 1 -- 172.21.15.92:0/2664018619 learned_addr learned my addr 172.21.15.92:0/2664018619 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:36.084 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.080+0000 7f77f2c55640 1 --2- 172.21.15.92:0/2664018619 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f77ec12ad40 0x7f77ec1bf800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:36.084 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.080+0000 7f77f2c55640 1 -- 172.21.15.92:0/2664018619 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f77ec126fa0 con 0x7f77d0003670 2024-09-21T09:41:36.084 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.080+0000 7f77f2c55640 1 -- 172.21.15.92:0/2664018619 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f77ec126e20 con 0x7f77ec12ad40 2024-09-21T09:41:36.084 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.080+0000 7f77f2c55640 1 -- 172.21.15.92:0/2664018619 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f77ec127120 con 0x7f77ec12a960 2024-09-21T09:41:36.084 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.080+0000 7f77f11f2640 1 --2- 172.21.15.92:0/2664018619 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f77ec12ad40 0x7f77ec1bf800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:36.084 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.081+0000 7f77f11f2640 1 -- 172.21.15.92:0/2664018619 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f77ec12a960 msgr2=0x7f77ec12eae0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.084 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.081+0000 7f77f11f2640 1 --2- 172.21.15.92:0/2664018619 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f77ec12a960 0x7f77ec12eae0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.085 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.081+0000 7f77f11f2640 1 -- 172.21.15.92:0/2664018619 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f77d0003670 msgr2=0x7f77ec12e5a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.085 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.081+0000 7f77f11f2640 1 --2- 172.21.15.92:0/2664018619 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f77d0003670 0x7f77ec12e5a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.085 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.081+0000 7f77f11f2640 1 -- 172.21.15.92:0/2664018619 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f77ec08b3c0 con 0x7f77ec12ad40 2024-09-21T09:41:36.085 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.081+0000 7f77ebfff640 1 --2- 172.21.15.92:0/2664018619 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f77ec12a960 0x7f77ec12eae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:36.085 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.081+0000 7f77f11f2640 1 --2- 172.21.15.92:0/2664018619 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f77ec12ad40 0x7f77ec1bf800 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f77ec075a60 tx=0x7f77e4013cb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:36.086 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.081+0000 7f77e97fa640 1 -- 172.21.15.92:0/2664018619 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f77e4004b20 con 0x7f77ec12ad40 2024-09-21T09:41:36.086 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.081+0000 7f77e97fa640 1 -- 172.21.15.92:0/2664018619 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f77e4004cc0 con 0x7f77ec12ad40 2024-09-21T09:41:36.086 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.081+0000 7f77f2c55640 1 -- 172.21.15.92:0/2664018619 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f77ec08bbc0 con 0x7f77ec12ad40 2024-09-21T09:41:36.086 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.081+0000 7f77e97fa640 1 -- 172.21.15.92:0/2664018619 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f77e4011d80 con 0x7f77ec12ad40 2024-09-21T09:41:36.086 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.082+0000 7f77e97fa640 1 -- 172.21.15.92:0/2664018619 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f77e400c040 con 0x7f77ec12ad40 2024-09-21T09:41:36.086 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.082+0000 7f77f2c55640 1 -- 172.21.15.92:0/2664018619 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f77ec12f200 con 0x7f77ec12ad40 2024-09-21T09:41:36.086 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.082+0000 7f77e97fa640 1 -- 172.21.15.92:0/2664018619 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f77e4005500 con 0x7f77ec12ad40 2024-09-21T09:41:36.087 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.083+0000 7f77d6ffd640 1 -- 172.21.15.92:0/2664018619 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f77c4005290 con 0x7f77ec12ad40 2024-09-21T09:41:36.090 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.087+0000 7f77e97fa640 1 -- 172.21.15.92:0/2664018619 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7f77e402c080 con 0x7f77ec12ad40 2024-09-21T09:41:36.102 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.099+0000 7f612d8eb640 1 -- 172.21.15.92:0/2443779737 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command([{prefix=config set, name=osd_op_complaint_time}] v 0) -- 0x7f6128068c30 con 0x7f612812b9b0 2024-09-21T09:41:36.106 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.104+0000 7f611ffff640 1 -- 172.21.15.92:0/2443779737 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{prefix=config set, name=osd_op_complaint_time}]=0 v16)=0 v16) ==== 121+0+0 (secure 0 0 0) 0x7f6128127330 con 0x7f612812b9b0 2024-09-21T09:41:36.108 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.105+0000 7f612d8eb640 1 -- 172.21.15.92:0/2443779737 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f612812b9b0 msgr2=0x7f61281251f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.108 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.105+0000 7f612d8eb640 1 --2- 172.21.15.92:0/2443779737 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f612812b9b0 0x7f61281251f0 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f61200062e0 tx=0x7f61200066e0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.108 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.105+0000 7f612d8eb640 1 -- 172.21.15.92:0/2443779737 shutdown_connections 2024-09-21T09:41:36.108 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.105+0000 7f612d8eb640 1 --2- 172.21.15.92:0/2443779737 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f612812b9b0 0x7f61281251f0 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.108 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.105+0000 7f612d8eb640 1 --2- 172.21.15.92:0/2443779737 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61281289b0 0x7f6128124cb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.108 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.105+0000 7f612d8eb640 1 --2- 172.21.15.92:0/2443779737 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6108003700 0x7f6128124770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.108 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.105+0000 7f612d8eb640 1 -- 172.21.15.92:0/2443779737 >> 172.21.15.92:0/2443779737 conn(0x7f612809e280 msgr2=0x7f612812a690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:36.108 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.105+0000 7f612d8eb640 1 -- 172.21.15.92:0/2443779737 shutdown_connections 2024-09-21T09:41:36.108 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.105+0000 7f612d8eb640 1 -- 172.21.15.92:0/2443779737 wait complete. 2024-09-21T09:41:36.119 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'rados_mon_op_timeout', '900'], check_status=True, hostname='smithi092') 2024-09-21T09:41:36.121 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.119+0000 7f9426965640 1 -- 172.21.15.92:0/3937719757 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command([{prefix=config set, name=rados_mon_op_timeout}] v 0) -- 0x7f9420068c30 con 0x7f94201289d0 2024-09-21T09:41:36.126 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.124+0000 7f941cff9640 1 -- 172.21.15.92:0/3937719757 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mon_command_ack([{prefix=config set, name=rados_mon_op_timeout}]=0 v17)=0 v17) ==== 120+0+0 (secure 0 0 0) 0x7f94140263f0 con 0x7f94201289d0 2024-09-21T09:41:36.128 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.125+0000 7f9426965640 1 -- 172.21.15.92:0/3937719757 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f94201289d0 msgr2=0x7f94201252b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.128 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.125+0000 7f9426965640 1 --2- 172.21.15.92:0/3937719757 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f94201289d0 0x7f94201252b0 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f94140059d0 tx=0x7f9414013c60 comp rx=0 tx=0).stop 2024-09-21T09:41:36.128 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.125+0000 7f9426965640 1 -- 172.21.15.92:0/3937719757 shutdown_connections 2024-09-21T09:41:36.128 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.125+0000 7f9426965640 1 --2- 172.21.15.92:0/3937719757 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f942012eee0 0x7f94201257f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.128 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.125+0000 7f9426965640 1 --2- 172.21.15.92:0/3937719757 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f94201289d0 0x7f94201252b0 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.128 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.125+0000 7f9426965640 1 --2- 172.21.15.92:0/3937719757 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f940c003670 0x7f9420124d70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.128 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.125+0000 7f9426965640 1 -- 172.21.15.92:0/3937719757 >> 172.21.15.92:0/3937719757 conn(0x7f942009d2b0 msgr2=0x7f942006fa60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:36.129 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.125+0000 7f9426965640 1 -- 172.21.15.92:0/3937719757 shutdown_connections 2024-09-21T09:41:36.129 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.125+0000 7f9426965640 1 -- 172.21.15.92:0/3937719757 wait complete. 2024-09-21T09:41:36.138 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'rados_osd_op_timeout', '900'], check_status=True, hostname='smithi092') 2024-09-21T09:41:36.183 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.180+0000 7f77d6ffd640 1 -- 172.21.15.92:0/2664018619 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command([{prefix=config set, name=rados_osd_op_timeout}] v 0) -- 0x7f77c4005850 con 0x7f77ec12ad40 2024-09-21T09:41:36.188 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.186+0000 7f77e97fa640 1 -- 172.21.15.92:0/2664018619 <== mon.1 v2:172.21.15.92:3301/0 7 ==== mon_command_ack([{prefix=config set, name=rados_osd_op_timeout}]=0 v18)=0 v18) ==== 120+0+0 (secure 0 0 0) 0x7f77e4005700 con 0x7f77ec12ad40 2024-09-21T09:41:36.189 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.187+0000 7f77d6ffd640 1 -- 172.21.15.92:0/2664018619 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f77ec12ad40 msgr2=0x7f77ec1bf800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.190 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.187+0000 7f77d6ffd640 1 --2- 172.21.15.92:0/2664018619 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f77ec12ad40 0x7f77ec1bf800 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f77ec075a60 tx=0x7f77e4013cb0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.190 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.187+0000 7f77d6ffd640 1 -- 172.21.15.92:0/2664018619 shutdown_connections 2024-09-21T09:41:36.190 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.187+0000 7f77d6ffd640 1 --2- 172.21.15.92:0/2664018619 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f77ec12ad40 0x7f77ec1bf800 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.190 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.187+0000 7f77d6ffd640 1 --2- 172.21.15.92:0/2664018619 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f77ec12a960 0x7f77ec12eae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.190 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.187+0000 7f77d6ffd640 1 --2- 172.21.15.92:0/2664018619 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f77d0003670 0x7f77ec12e5a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.190 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.187+0000 7f77d6ffd640 1 -- 172.21.15.92:0/2664018619 >> 172.21.15.92:0/2664018619 conn(0x7f77ec09c6c0 msgr2=0x7f77ec06de20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:36.190 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.187+0000 7f77d6ffd640 1 -- 172.21.15.92:0/2664018619 shutdown_connections 2024-09-21T09:41:36.190 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.187+0000 7f77d6ffd640 1 -- 172.21.15.92:0/2664018619 wait complete. 2024-09-21T09:41:36.199 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mon', 'mon_op_complaint_time', '120'], check_status=True, hostname='smithi092') 2024-09-21T09:41:36.199 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mon', 'mon_warn_on_pool_no_app', 'False'], check_status=True, hostname='smithi092') 2024-09-21T09:41:36.200 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'osd', 'osd_op_complaint_time', '180'], check_status=True, hostname='smithi092') 2024-09-21T09:41:36.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.205+0000 7f0c866eb640 1 -- 172.21.15.92:0/2739973341 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0c8012b140 msgr2=0x7f0c6c00a8f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.205+0000 7f0c866eb640 1 --2- 172.21.15.92:0/2739973341 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0c8012b140 0x7f0c6c00a8f0 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f0c74005ce0 tx=0x7f0c7402bbd0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.205+0000 7f0c866eb640 1 -- 172.21.15.92:0/2739973341 shutdown_connections 2024-09-21T09:41:36.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.205+0000 7f0c866eb640 1 --2- 172.21.15.92:0/2739973341 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0c6c003670 0x7f0c6c00aea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.205+0000 7f0c866eb640 1 --2- 172.21.15.92:0/2739973341 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0c8012b140 0x7f0c6c00a8f0 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.205+0000 7f0c866eb640 1 --2- 172.21.15.92:0/2739973341 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0c80133430 0x7f0c6c008380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.206+0000 7f0c866eb640 1 -- 172.21.15.92:0/2739973341 >> 172.21.15.92:0/2739973341 conn(0x7f0c8006db60 msgr2=0x7f0c8006df70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:36.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.206+0000 7f0c866eb640 1 -- 172.21.15.92:0/2739973341 shutdown_connections 2024-09-21T09:41:36.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.206+0000 7f0c866eb640 1 -- 172.21.15.92:0/2739973341 wait complete. 2024-09-21T09:41:36.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.207+0000 7f0c866eb640 1 Processor -- start 2024-09-21T09:41:36.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.207+0000 7f0c866eb640 1 -- start start 2024-09-21T09:41:36.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.208+0000 7f0c866eb640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0c6c003670 0x7f0c801c3740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:36.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.208+0000 7f0c866eb640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0c8012a650 0x7f0c801c3c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:36.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.208+0000 7f0c866eb640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0c8012b140 0x7f0c801c41c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:36.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.208+0000 7f0c866eb640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f0c80131b70 con 0x7f0c8012a650 2024-09-21T09:41:36.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.208+0000 7f0c866eb640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f0c801319f0 con 0x7f0c8012b140 2024-09-21T09:41:36.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.208+0000 7f0c866eb640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f0c80131cf0 con 0x7f0c6c003670 2024-09-21T09:41:36.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.208+0000 7f0c7ffff640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0c6c003670 0x7f0c801c3740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:36.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.208+0000 7f0c7f7fe640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0c8012a650 0x7f0c801c3c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:36.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.208+0000 7f0c7ffff640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0c6c003670 0x7f0c801c3740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:34716/0 (socket says 172.21.15.92:34716) 2024-09-21T09:41:36.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.208+0000 7f0c7ffff640 1 -- 172.21.15.92:0/1188915957 learned_addr learned my addr 172.21.15.92:0/1188915957 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:36.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.208+0000 7f0c84c88640 1 --2- 172.21.15.92:0/1188915957 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0c8012b140 0x7f0c801c41c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:36.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.208+0000 7f0c7f7fe640 1 -- 172.21.15.92:0/1188915957 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0c6c003670 msgr2=0x7f0c801c3740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.208+0000 7f0c7f7fe640 1 --2- 172.21.15.92:0/1188915957 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0c6c003670 0x7f0c801c3740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.208+0000 7f0c7f7fe640 1 -- 172.21.15.92:0/1188915957 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0c8012b140 msgr2=0x7f0c801c41c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.208+0000 7f0c7f7fe640 1 --2- 172.21.15.92:0/1188915957 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0c8012b140 0x7f0c801c41c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.214 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.208+0000 7f0c7f7fe640 1 -- 172.21.15.92:0/1188915957 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0c801275b0 con 0x7f0c8012a650 2024-09-21T09:41:36.214 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.208+0000 7f0c84c88640 1 --2- 172.21.15.92:0/1188915957 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0c8012b140 0x7f0c801c41c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:36.214 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.208+0000 7f0c7f7fe640 1 --2- 172.21.15.92:0/1188915957 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0c8012a650 0x7f0c801c3c80 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f0c8009d000 tx=0x7f0c740376a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:36.214 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.209+0000 7f0c7cff9640 1 -- 172.21.15.92:0/1188915957 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0c74037aa0 con 0x7f0c8012a650 2024-09-21T09:41:36.214 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.209+0000 7f0c7cff9640 1 -- 172.21.15.92:0/1188915957 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f0c74045070 con 0x7f0c8012a650 2024-09-21T09:41:36.214 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.209+0000 7f0c866eb640 1 -- 172.21.15.92:0/1188915957 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0c80119580 con 0x7f0c8012a650 2024-09-21T09:41:36.214 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.209+0000 7f0c7cff9640 1 -- 172.21.15.92:0/1188915957 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0c74040480 con 0x7f0c8012a650 2024-09-21T09:41:36.214 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.209+0000 7f0c866eb640 1 -- 172.21.15.92:0/1188915957 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0c80123b70 con 0x7f0c8012a650 2024-09-21T09:41:36.214 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.209+0000 7f0c7cff9640 1 -- 172.21.15.92:0/1188915957 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f0c74040620 con 0x7f0c8012a650 2024-09-21T09:41:36.214 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.209+0000 7f0c7cff9640 1 -- 172.21.15.92:0/1188915957 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7f0c7404a460 con 0x7f0c8012a650 2024-09-21T09:41:36.215 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.210+0000 7f0c866eb640 1 -- 172.21.15.92:0/1188915957 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0c8012aa30 con 0x7f0c8012a650 2024-09-21T09:41:36.215 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.212+0000 7f0c7cff9640 1 -- 172.21.15.92:0/1188915957 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7f0c74040a50 con 0x7f0c8012a650 2024-09-21T09:41:36.273 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.270+0000 7f0c866eb640 1 -- 172.21.15.92:0/1188915957 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command([{prefix=config set, name=osd_op_complaint_time}] v 0) -- 0x7f0c80073ee0 con 0x7f0c8012a650 2024-09-21T09:41:36.278 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.275+0000 7f0c7cff9640 1 -- 172.21.15.92:0/1188915957 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{prefix=config set, name=osd_op_complaint_time}]=0 v19)=0 v19) ==== 121+0+0 (secure 0 0 0) 0x7f0c74040c30 con 0x7f0c8012a650 2024-09-21T09:41:36.279 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.276+0000 7f0c866eb640 1 -- 172.21.15.92:0/1188915957 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0c8012a650 msgr2=0x7f0c801c3c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.280 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.276+0000 7f0c866eb640 1 --2- 172.21.15.92:0/1188915957 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0c8012a650 0x7f0c801c3c80 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f0c8009d000 tx=0x7f0c740376a0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.280 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.277+0000 7f0c866eb640 1 -- 172.21.15.92:0/1188915957 shutdown_connections 2024-09-21T09:41:36.280 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.277+0000 7f0c866eb640 1 --2- 172.21.15.92:0/1188915957 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0c8012b140 0x7f0c801c41c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.280 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.277+0000 7f0c866eb640 1 --2- 172.21.15.92:0/1188915957 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0c8012a650 0x7f0c801c3c80 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.280 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.277+0000 7f0c866eb640 1 --2- 172.21.15.92:0/1188915957 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0c6c003670 0x7f0c801c3740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.280 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.277+0000 7f0c866eb640 1 -- 172.21.15.92:0/1188915957 >> 172.21.15.92:0/1188915957 conn(0x7f0c8006db60 msgr2=0x7f0c8009e420 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:36.280 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.277+0000 7f0c866eb640 1 -- 172.21.15.92:0/1188915957 shutdown_connections 2024-09-21T09:41:36.280 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.277+0000 7f0c866eb640 1 -- 172.21.15.92:0/1188915957 wait complete. 2024-09-21T09:41:36.289 INFO:tasks.ceph:Setting crush tunables to default 2024-09-21T09:41:36.289 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph osd crush tunables default 2024-09-21T09:41:36.295 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:36.293+0000 7ffbab4a0040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:41:36.296 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:36.293+0000 7fdc77016040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:41:36.350 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:36.347+0000 7ffbab4a0040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:41:36.350 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:36.347+0000 7fdc77016040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:41:36.396 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.393+0000 7fe0a5fa0640 1 -- 172.21.15.92:0/870779207 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe0a012bcc0 msgr2=0x7fe0a012c0a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.396 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.393+0000 7fe0a5fa0640 1 --2- 172.21.15.92:0/870779207 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe0a012bcc0 0x7fe0a012c0a0 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7fe094009900 tx=0x7fe09402a580 comp rx=0 tx=0).stop 2024-09-21T09:41:36.396 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.393+0000 7fe0a5fa0640 1 -- 172.21.15.92:0/870779207 shutdown_connections 2024-09-21T09:41:36.397 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.393+0000 7fe0a5fa0640 1 --2- 172.21.15.92:0/870779207 >> v2:172.21.15.92:3300/0 conn(0x7fe0a009a320 0x7fe0a0121f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.397 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.393+0000 7fe0a5fa0640 1 --2- 172.21.15.92:0/870779207 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe0a012bcc0 0x7fe0a012c0a0 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.397 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.393+0000 7fe0a5fa0640 1 -- 172.21.15.92:0/870779207 >> 172.21.15.92:0/870779207 conn(0x7fe0a009d4d0 msgr2=0x7fe0a00999a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:36.397 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.393+0000 7fe0a5fa0640 1 -- 172.21.15.92:0/870779207 shutdown_connections 2024-09-21T09:41:36.397 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.393+0000 7fe0a5fa0640 1 -- 172.21.15.92:0/870779207 wait complete. 2024-09-21T09:41:36.399 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.394+0000 7fe0a5fa0640 1 Processor -- start 2024-09-21T09:41:36.399 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.394+0000 7fe0a5fa0640 1 -- start start 2024-09-21T09:41:36.399 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.395+0000 7fe0a5fa0640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe0a009a320 0x7fe0a01c1c70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:36.399 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.395+0000 7fe0a5fa0640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe0a01224a0 0x7fe0a01c21b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:36.399 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.395+0000 7fe0a5fa0640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe0a012bcc0 0x7fe0a0123df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:36.399 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.395+0000 7fe0a5fa0640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fe0a0134690 con 0x7fe0a01224a0 2024-09-21T09:41:36.399 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.395+0000 7fe0a5fa0640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fe0a0134510 con 0x7fe0a009a320 2024-09-21T09:41:36.399 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.395+0000 7fe0a5fa0640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fe0a0134810 con 0x7fe0a012bcc0 2024-09-21T09:41:36.400 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.395+0000 7fe0a4f9e640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe0a009a320 0x7fe0a01c1c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:36.400 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.395+0000 7fe0a579f640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe0a012bcc0 0x7fe0a0123df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:36.400 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.395+0000 7fe09ffff640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe0a01224a0 0x7fe0a01c21b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:36.400 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.395+0000 7fe0a579f640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe0a012bcc0 0x7fe0a0123df0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:34734/0 (socket says 172.21.15.92:34734) 2024-09-21T09:41:36.400 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.395+0000 7fe0a579f640 1 -- 172.21.15.92:0/1524535827 learned_addr learned my addr 172.21.15.92:0/1524535827 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:36.400 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.395+0000 7fe0a4f9e640 1 -- 172.21.15.92:0/1524535827 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe0a012bcc0 msgr2=0x7fe0a0123df0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.400 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.395+0000 7fe0a4f9e640 1 --2- 172.21.15.92:0/1524535827 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe0a012bcc0 0x7fe0a0123df0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.400 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.395+0000 7fe0a4f9e640 1 -- 172.21.15.92:0/1524535827 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe0a01224a0 msgr2=0x7fe0a01c21b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.400 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.395+0000 7fe0a4f9e640 1 --2- 172.21.15.92:0/1524535827 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe0a01224a0 0x7fe0a01c21b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.400 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.395+0000 7fe0a4f9e640 1 -- 172.21.15.92:0/1524535827 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe0a012ddf0 con 0x7fe0a009a320 2024-09-21T09:41:36.400 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.395+0000 7fe0a579f640 1 --2- 172.21.15.92:0/1524535827 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe0a012bcc0 0x7fe0a0123df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:36.400 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.395+0000 7fe09ffff640 1 --2- 172.21.15.92:0/1524535827 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe0a01224a0 0x7fe0a01c21b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:36.400 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.396+0000 7fe0a4f9e640 1 --2- 172.21.15.92:0/1524535827 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe0a009a320 0x7fe0a01c1c70 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7fe094031000 tx=0x7fe0940038a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:36.400 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.396+0000 7fe09d7fa640 1 -- 172.21.15.92:0/1524535827 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe09403f070 con 0x7fe0a009a320 2024-09-21T09:41:36.401 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.396+0000 7fe09d7fa640 1 -- 172.21.15.92:0/1524535827 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fe0940047a0 con 0x7fe0a009a320 2024-09-21T09:41:36.401 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.396+0000 7fe0a5fa0640 1 -- 172.21.15.92:0/1524535827 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe0a012da20 con 0x7fe0a009a320 2024-09-21T09:41:36.401 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.396+0000 7fe09d7fa640 1 -- 172.21.15.92:0/1524535827 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe094035030 con 0x7fe0a009a320 2024-09-21T09:41:36.401 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.396+0000 7fe0a5fa0640 1 -- 172.21.15.92:0/1524535827 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fe0a012d350 con 0x7fe0a009a320 2024-09-21T09:41:36.401 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.396+0000 7fe09d7fa640 1 -- 172.21.15.92:0/1524535827 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7fe094042070 con 0x7fe0a009a320 2024-09-21T09:41:36.401 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.396+0000 7fe09d7fa640 1 -- 172.21.15.92:0/1524535827 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(1..1 src has 1..1) ==== 725+0+0 (secure 0 0 0) 0x7fe09403b8d0 con 0x7fe0a009a320 2024-09-21T09:41:36.401 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.397+0000 7fe0a5fa0640 1 -- 172.21.15.92:0/1524535827 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe0a0121f60 con 0x7fe0a009a320 2024-09-21T09:41:36.401 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.399+0000 7fe09d7fa640 1 -- 172.21.15.92:0/1524535827 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7fe09403bb20 con 0x7fe0a009a320 2024-09-21T09:41:36.464 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.461+0000 7fe0a5fa0640 1 -- 172.21.15.92:0/1524535827 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "osd crush tunables", "profile": "default"} v 0) -- 0x7fe0a0068c30 con 0x7fe0a009a320 2024-09-21T09:41:36.468 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.466+0000 7fe09d7fa640 1 -- 172.21.15.92:0/1524535827 <== mon.1 v2:172.21.15.92:3301/0 7 ==== mon_command_ack([{"prefix": "osd crush tunables", "profile": "default"}]=0 adjusted tunables profile to default v2) ==== 124+0+0 (secure 0 0 0) 0x7fe09403bd00 con 0x7fe0a009a320 2024-09-21T09:41:36.468 INFO:teuthology.orchestra.run.smithi092.stderr:adjusted tunables profile to default 2024-09-21T09:41:36.469 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.466+0000 7fe0a5fa0640 1 -- 172.21.15.92:0/1524535827 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe0a009a320 msgr2=0x7fe0a01c1c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.469 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.466+0000 7fe0a5fa0640 1 --2- 172.21.15.92:0/1524535827 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe0a009a320 0x7fe0a01c1c70 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7fe094031000 tx=0x7fe0940038a0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.469 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.467+0000 7fe0a5fa0640 1 -- 172.21.15.92:0/1524535827 shutdown_connections 2024-09-21T09:41:36.470 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.467+0000 7fe0a5fa0640 1 --2- 172.21.15.92:0/1524535827 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe0a012bcc0 0x7fe0a0123df0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.470 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.467+0000 7fe0a5fa0640 1 --2- 172.21.15.92:0/1524535827 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe0a01224a0 0x7fe0a01c21b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.470 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.467+0000 7fe0a5fa0640 1 --2- 172.21.15.92:0/1524535827 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe0a009a320 0x7fe0a01c1c70 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.470 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.467+0000 7fe0a5fa0640 1 -- 172.21.15.92:0/1524535827 >> 172.21.15.92:0/1524535827 conn(0x7fe0a009d4d0 msgr2=0x7fe0a01263c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:36.470 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.467+0000 7fe0a5fa0640 1 -- 172.21.15.92:0/1524535827 shutdown_connections 2024-09-21T09:41:36.470 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.467+0000 7fe0a5fa0640 1 -- 172.21.15.92:0/1524535827 wait complete. 2024-09-21T09:41:36.480 INFO:tasks.ceph:check_enable_crimson: False 2024-09-21T09:41:36.480 INFO:tasks.ceph:Starting osd daemons in cluster ceph... 2024-09-21T09:41:36.480 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:41:36.480 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd if=/var/lib/ceph/osd/ceph-0/fsid of=/dev/stdout 2024-09-21T09:41:36.497 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:36.495+0000 7fdc77016040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:41:36.498 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:36.495+0000 7ffbab4a0040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:41:36.504 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:41:36.504 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd if=/var/lib/ceph/osd/ceph-1/fsid of=/dev/stdout 2024-09-21T09:41:36.569 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:41:36.569 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd if=/var/lib/ceph/osd/ceph-2/fsid of=/dev/stdout 2024-09-21T09:41:36.634 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:41:36.634 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd if=/var/lib/ceph/osd/ceph-3/fsid of=/dev/stdout 2024-09-21T09:41:36.699 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph osd new f3314f59-3374-4072-85c9-f8c4e458efc8 0 2024-09-21T09:41:36.839 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.836+0000 7ff5b92eb640 1 -- 172.21.15.92:0/1987061331 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff5b406bbc0 msgr2=0x7ff5b406bfa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.839 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.836+0000 7ff5b92eb640 1 --2- 172.21.15.92:0/1987061331 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff5b406bbc0 0x7ff5b406bfa0 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7ff5a4009860 tx=0x7ff5a402a380 comp rx=0 tx=0).stop 2024-09-21T09:41:36.839 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.836+0000 7ff5b92eb640 1 -- 172.21.15.92:0/1987061331 shutdown_connections 2024-09-21T09:41:36.840 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.836+0000 7ff5b92eb640 1 --2- 172.21.15.92:0/1987061331 >> v2:172.21.15.92:3300/0 conn(0x7ff5b4130170 0x7ff5b4132560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.840 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.836+0000 7ff5b92eb640 1 --2- 172.21.15.92:0/1987061331 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff5b40957b0 0x7ff5b406c4e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.840 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.836+0000 7ff5b92eb640 1 --2- 172.21.15.92:0/1987061331 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff5b406bbc0 0x7ff5b406bfa0 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.840 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.836+0000 7ff5b92eb640 1 -- 172.21.15.92:0/1987061331 >> 172.21.15.92:0/1987061331 conn(0x7ff5b409d620 msgr2=0x7ff5b4099af0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:36.840 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.836+0000 7ff5b92eb640 1 -- 172.21.15.92:0/1987061331 shutdown_connections 2024-09-21T09:41:36.840 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.836+0000 7ff5b92eb640 1 -- 172.21.15.92:0/1987061331 wait complete. 2024-09-21T09:41:36.842 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.837+0000 7ff5b92eb640 1 Processor -- start 2024-09-21T09:41:36.843 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.837+0000 7ff5b92eb640 1 -- start start 2024-09-21T09:41:36.843 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.837+0000 7ff5b92eb640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff5b406bbc0 0x7ff5b412f7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:36.843 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.837+0000 7ff5b92eb640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff5b40957b0 0x7ff5b406e9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:36.843 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.837+0000 7ff5b92eb640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff5b4130170 0x7ff5b406eef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:36.843 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.837+0000 7ff5b92eb640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7ff5b4133810 con 0x7ff5b40957b0 2024-09-21T09:41:36.843 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.837+0000 7ff5b92eb640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7ff5b4133690 con 0x7ff5b4130170 2024-09-21T09:41:36.843 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.837+0000 7ff5b92eb640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7ff5b4133990 con 0x7ff5b406bbc0 2024-09-21T09:41:36.843 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.837+0000 7ff5b27fc640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff5b40957b0 0x7ff5b406e9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:36.843 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.837+0000 7ff5b2ffd640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff5b406bbc0 0x7ff5b412f7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:36.843 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.837+0000 7ff5b27fc640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff5b40957b0 0x7ff5b406e9b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:39500/0 (socket says 172.21.15.92:39500) 2024-09-21T09:41:36.844 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.837+0000 7ff5b2ffd640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff5b406bbc0 0x7ff5b412f7c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:34762/0 (socket says 172.21.15.92:34762) 2024-09-21T09:41:36.844 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.837+0000 7ff5b27fc640 1 -- 172.21.15.92:0/4074778784 learned_addr learned my addr 172.21.15.92:0/4074778784 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:36.844 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.838+0000 7ff5b37fe640 1 --2- 172.21.15.92:0/4074778784 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff5b4130170 0x7ff5b406eef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:36.844 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.838+0000 7ff5b2ffd640 1 -- 172.21.15.92:0/4074778784 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff5b4130170 msgr2=0x7ff5b406eef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.844 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.838+0000 7ff5b2ffd640 1 --2- 172.21.15.92:0/4074778784 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff5b4130170 0x7ff5b406eef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.844 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.838+0000 7ff5b2ffd640 1 -- 172.21.15.92:0/4074778784 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff5b40957b0 msgr2=0x7ff5b406e9b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.844 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.838+0000 7ff5b2ffd640 1 --2- 172.21.15.92:0/4074778784 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff5b40957b0 0x7ff5b406e9b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.844 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.838+0000 7ff5b2ffd640 1 -- 172.21.15.92:0/4074778784 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff5b4092200 con 0x7ff5b406bbc0 2024-09-21T09:41:36.844 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.838+0000 7ff5b27fc640 1 --2- 172.21.15.92:0/4074778784 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff5b40957b0 0x7ff5b406e9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:36.844 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.838+0000 7ff5b2ffd640 1 --2- 172.21.15.92:0/4074778784 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff5b406bbc0 0x7ff5b412f7c0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7ff5b409adf0 tx=0x7ff5a4033b70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:36.844 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.838+0000 7ff59b7fe640 1 -- 172.21.15.92:0/4074778784 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff5a403f020 con 0x7ff5b406bbc0 2024-09-21T09:41:36.845 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.838+0000 7ff5b92eb640 1 -- 172.21.15.92:0/4074778784 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff5b4090e20 con 0x7ff5b406bbc0 2024-09-21T09:41:36.845 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.838+0000 7ff59b7fe640 1 -- 172.21.15.92:0/4074778784 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7ff5a4033d60 con 0x7ff5b406bbc0 2024-09-21T09:41:36.845 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.838+0000 7ff59b7fe640 1 -- 172.21.15.92:0/4074778784 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff5a4037040 con 0x7ff5b406bbc0 2024-09-21T09:41:36.845 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.838+0000 7ff5b92eb640 1 -- 172.21.15.92:0/4074778784 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7ff5b412fd00 con 0x7ff5b406bbc0 2024-09-21T09:41:36.845 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.838+0000 7ff59b7fe640 1 -- 172.21.15.92:0/4074778784 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7ff5a4042020 con 0x7ff5b406bbc0 2024-09-21T09:41:36.845 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.838+0000 7ff59b7fe640 1 -- 172.21.15.92:0/4074778784 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(2..2 src has 1..2) ==== 725+0+0 (secure 0 0 0) 0x7ff5a4044990 con 0x7ff5b406bbc0 2024-09-21T09:41:36.846 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:36.839+0000 7fdc77016040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:41:36.847 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:36.839+0000 7ffbab4a0040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:41:36.847 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.839+0000 7ff5b92eb640 1 -- 172.21.15.92:0/4074778784 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff5b406bfa0 con 0x7ff5b406bbc0 2024-09-21T09:41:36.847 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.841+0000 7ff59b7fe640 1 -- 172.21.15.92:0/4074778784 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7ff5a4049030 con 0x7ff5b406bbc0 2024-09-21T09:41:36.896 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:36.894+0000 7fdc77016040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:41:36.908 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:36.906+0000 7ffbab4a0040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:41:36.943 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.940+0000 7ff5b92eb640 1 -- 172.21.15.92:0/4074778784 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "osd new", "uuid": "f3314f59-3374-4072-85c9-f8c4e458efc8", "id": 0} v 0) -- 0x7ff5b4068c30 con 0x7ff5b406bbc0 2024-09-21T09:41:36.946 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.944+0000 7ff59b7fe640 1 -- 172.21.15.92:0/4074778784 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "f3314f59-3374-4072-85c9-f8c4e458efc8", "id": 0}]=0 v3) ==== 112+0+2 (secure 0 0 0) 0x7ff5a4003dc0 con 0x7ff5b406bbc0 2024-09-21T09:41:36.946 INFO:teuthology.orchestra.run.smithi092.stdout:0 2024-09-21T09:41:36.948 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.945+0000 7ff5b92eb640 1 -- 172.21.15.92:0/4074778784 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff5b406bbc0 msgr2=0x7ff5b412f7c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:36.948 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.945+0000 7ff5b92eb640 1 --2- 172.21.15.92:0/4074778784 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff5b406bbc0 0x7ff5b412f7c0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7ff5b409adf0 tx=0x7ff5a4033b70 comp rx=0 tx=0).stop 2024-09-21T09:41:36.948 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.945+0000 7ff5b92eb640 1 -- 172.21.15.92:0/4074778784 shutdown_connections 2024-09-21T09:41:36.948 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.945+0000 7ff5b92eb640 1 --2- 172.21.15.92:0/4074778784 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff5b4130170 0x7ff5b406eef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.948 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.945+0000 7ff5b92eb640 1 --2- 172.21.15.92:0/4074778784 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff5b40957b0 0x7ff5b406e9b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.948 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.945+0000 7ff5b92eb640 1 --2- 172.21.15.92:0/4074778784 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff5b406bbc0 0x7ff5b412f7c0 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:36.948 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.945+0000 7ff5b92eb640 1 -- 172.21.15.92:0/4074778784 >> 172.21.15.92:0/4074778784 conn(0x7ff5b409d620 msgr2=0x7ff5b4130fa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:36.948 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.945+0000 7ff5b92eb640 1 -- 172.21.15.92:0/4074778784 shutdown_connections 2024-09-21T09:41:36.948 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:36.945+0000 7ff5b92eb640 1 -- 172.21.15.92:0/4074778784 wait complete. 2024-09-21T09:41:36.958 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph osd new cf0486af-ba68-4763-a364-c1f2181604a7 1 2024-09-21T09:41:36.986 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:36.983+0000 7fdc77016040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:41:36.996 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:36.994+0000 7ffbab4a0040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:41:37.059 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.055+0000 7f98b35a5640 1 -- 172.21.15.92:0/2163790458 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f98ac131800 msgr2=0x7f98ac133bf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:37.059 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.055+0000 7f98b35a5640 1 --2- 172.21.15.92:0/2163790458 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f98ac131800 0x7f98ac133bf0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f98a4005fd0 tx=0x7f98a402a980 comp rx=0 tx=0).stop 2024-09-21T09:41:37.059 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.055+0000 7f98b35a5640 1 -- 172.21.15.92:0/2163790458 shutdown_connections 2024-09-21T09:41:37.059 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.055+0000 7f98b35a5640 1 --2- 172.21.15.92:0/2163790458 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f98ac131800 0x7f98ac133bf0 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.059 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.055+0000 7f98b35a5640 1 --2- 172.21.15.92:0/2163790458 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f98ac12eed0 0x7f98ac1312c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.059 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.055+0000 7f98b35a5640 1 -- 172.21.15.92:0/2163790458 >> 172.21.15.92:0/2163790458 conn(0x7f98ac09e590 msgr2=0x7f98ac09e9a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:37.059 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.056+0000 7f98b35a5640 1 -- 172.21.15.92:0/2163790458 shutdown_connections 2024-09-21T09:41:37.059 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.056+0000 7f98b35a5640 1 -- 172.21.15.92:0/2163790458 wait complete. 2024-09-21T09:41:37.061 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.056+0000 7f98b35a5640 1 Processor -- start 2024-09-21T09:41:37.061 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.056+0000 7f98b35a5640 1 -- start start 2024-09-21T09:41:37.061 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.057+0000 7f98b35a5640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f98ac095e20 0x7f98ac093f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:37.061 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.057+0000 7f98b35a5640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f98ac12eed0 0x7f98ac0944a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:37.061 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.057+0000 7f98b35a5640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f98ac131800 0x7f98ac0949e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:37.061 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.057+0000 7f98b35a5640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f98ac1349d0 con 0x7f98ac131800 2024-09-21T09:41:37.061 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.057+0000 7f98b35a5640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f98ac134850 con 0x7f98ac095e20 2024-09-21T09:41:37.062 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.057+0000 7f98b35a5640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f98ac134b50 con 0x7f98ac12eed0 2024-09-21T09:41:37.062 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.057+0000 7f98b0b40640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f98ac12eed0 0x7f98ac0944a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:37.062 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.057+0000 7f98b1b42640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f98ac131800 0x7f98ac0949e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:37.062 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.057+0000 7f98b1b42640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f98ac131800 0x7f98ac0949e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:39516/0 (socket says 172.21.15.92:39516) 2024-09-21T09:41:37.062 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.057+0000 7f98b0b40640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f98ac12eed0 0x7f98ac0944a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:34778/0 (socket says 172.21.15.92:34778) 2024-09-21T09:41:37.062 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.057+0000 7f98b1b42640 1 -- 172.21.15.92:0/4059584320 learned_addr learned my addr 172.21.15.92:0/4059584320 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:37.062 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.057+0000 7f98b1341640 1 --2- 172.21.15.92:0/4059584320 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f98ac095e20 0x7f98ac093f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:37.062 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.057+0000 7f98b0b40640 1 -- 172.21.15.92:0/4059584320 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f98ac095e20 msgr2=0x7f98ac093f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:37.062 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.057+0000 7f98b0b40640 1 --2- 172.21.15.92:0/4059584320 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f98ac095e20 0x7f98ac093f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.062 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.057+0000 7f98b0b40640 1 -- 172.21.15.92:0/4059584320 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f98ac131800 msgr2=0x7f98ac0949e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:37.062 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.057+0000 7f98b0b40640 1 --2- 172.21.15.92:0/4059584320 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f98ac131800 0x7f98ac0949e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.062 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.057+0000 7f98b0b40640 1 -- 172.21.15.92:0/4059584320 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f98ac06ebc0 con 0x7f98ac12eed0 2024-09-21T09:41:37.062 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.057+0000 7f98b1b42640 1 --2- 172.21.15.92:0/4059584320 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f98ac131800 0x7f98ac0949e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:37.063 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.058+0000 7f98b0b40640 1 --2- 172.21.15.92:0/4059584320 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f98ac12eed0 0x7f98ac0944a0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f989c00ea20 tx=0x7f989c00eee0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:37.063 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.058+0000 7f98a1ffb640 1 -- 172.21.15.92:0/4059584320 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f989c00cd50 con 0x7f98ac12eed0 2024-09-21T09:41:37.063 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.058+0000 7f98a1ffb640 1 -- 172.21.15.92:0/4059584320 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f989c004020 con 0x7f98ac12eed0 2024-09-21T09:41:37.063 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.058+0000 7f98b35a5640 1 -- 172.21.15.92:0/4059584320 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f98ac126470 con 0x7f98ac12eed0 2024-09-21T09:41:37.063 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.058+0000 7f98a1ffb640 1 -- 172.21.15.92:0/4059584320 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f989c004450 con 0x7f98ac12eed0 2024-09-21T09:41:37.063 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.058+0000 7f98a1ffb640 1 -- 172.21.15.92:0/4059584320 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f989c003980 con 0x7f98ac12eed0 2024-09-21T09:41:37.063 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.058+0000 7f98b35a5640 1 -- 172.21.15.92:0/4059584320 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f98ac06e390 con 0x7f98ac12eed0 2024-09-21T09:41:37.063 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.058+0000 7f98a1ffb640 1 -- 172.21.15.92:0/4059584320 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(3..3 src has 1..3) ==== 840+0+0 (secure 0 0 0) 0x7f989c0187e0 con 0x7f98ac12eed0 2024-09-21T09:41:37.064 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.059+0000 7f98b35a5640 1 -- 172.21.15.92:0/4059584320 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f98ac096200 con 0x7f98ac12eed0 2024-09-21T09:41:37.064 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.061+0000 7f98a1ffb640 1 -- 172.21.15.92:0/4059584320 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7f989c026070 con 0x7f98ac12eed0 2024-09-21T09:41:37.085 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:37.083+0000 7fdc77016040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:41:37.096 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:37.093+0000 7ffbab4a0040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:41:37.133 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:37.130+0000 7fdc77016040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:41:37.134 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.131+0000 7f98b35a5640 1 -- 172.21.15.92:0/4059584320 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "osd new", "uuid": "cf0486af-ba68-4763-a364-c1f2181604a7", "id": 1} v 0) -- 0x7f98ac073ee0 con 0x7f98ac12eed0 2024-09-21T09:41:37.138 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.135+0000 7f98a1ffb640 1 -- 172.21.15.92:0/4059584320 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "cf0486af-ba68-4763-a364-c1f2181604a7", "id": 1}]=0 v4) ==== 112+0+2 (secure 0 0 0) 0x7f98ac06ebc0 con 0x7f98ac12eed0 2024-09-21T09:41:37.138 INFO:teuthology.orchestra.run.smithi092.stdout:1 2024-09-21T09:41:37.139 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.136+0000 7f98b35a5640 1 -- 172.21.15.92:0/4059584320 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f98ac12eed0 msgr2=0x7f98ac0944a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:37.139 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.136+0000 7f98b35a5640 1 --2- 172.21.15.92:0/4059584320 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f98ac12eed0 0x7f98ac0944a0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f989c00ea20 tx=0x7f989c00eee0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.139 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.136+0000 7f98b35a5640 1 -- 172.21.15.92:0/4059584320 shutdown_connections 2024-09-21T09:41:37.139 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.136+0000 7f98b35a5640 1 --2- 172.21.15.92:0/4059584320 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f98ac131800 0x7f98ac0949e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.139 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.136+0000 7f98b35a5640 1 --2- 172.21.15.92:0/4059584320 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f98ac12eed0 0x7f98ac0944a0 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.139 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.136+0000 7f98b35a5640 1 --2- 172.21.15.92:0/4059584320 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f98ac095e20 0x7f98ac093f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.140 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.136+0000 7f98b35a5640 1 -- 172.21.15.92:0/4059584320 >> 172.21.15.92:0/4059584320 conn(0x7f98ac09e590 msgr2=0x7f98ac131be0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:37.140 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.136+0000 7f98b35a5640 1 -- 172.21.15.92:0/4059584320 shutdown_connections 2024-09-21T09:41:37.140 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.136+0000 7f98b35a5640 1 -- 172.21.15.92:0/4059584320 wait complete. 2024-09-21T09:41:37.140 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:37.138+0000 7ffbab4a0040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:41:37.150 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph osd new 5c587421-3409-4469-a2ee-b2ea20cd45e5 2 2024-09-21T09:41:37.239 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:37.236+0000 7fdc77016040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:41:37.244 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:37.241+0000 7ffbab4a0040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:41:37.291 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.288+0000 7f1e5856c640 1 -- 172.21.15.92:0/1494039899 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1e50129f90 msgr2=0x7f1e5012a370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:37.291 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.288+0000 7f1e5856c640 1 --2- 172.21.15.92:0/1494039899 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1e50129f90 0x7f1e5012a370 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f1e4c009860 tx=0x7f1e4c02a3d0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.291 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.288+0000 7f1e5856c640 1 -- 172.21.15.92:0/1494039899 shutdown_connections 2024-09-21T09:41:37.291 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.288+0000 7f1e5856c640 1 --2- 172.21.15.92:0/1494039899 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f1e50094ff0 0x7f1e50131f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.291 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.288+0000 7f1e5856c640 1 --2- 172.21.15.92:0/1494039899 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1e50129f90 0x7f1e5012a370 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.291 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.288+0000 7f1e5856c640 1 -- 172.21.15.92:0/1494039899 >> 172.21.15.92:0/1494039899 conn(0x7f1e5009ca00 msgr2=0x7f1e50098ed0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:37.292 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.288+0000 7f1e5856c640 1 -- 172.21.15.92:0/1494039899 shutdown_connections 2024-09-21T09:41:37.292 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.288+0000 7f1e5856c640 1 -- 172.21.15.92:0/1494039899 wait complete. 2024-09-21T09:41:37.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.289+0000 7f1e5856c640 1 Processor -- start 2024-09-21T09:41:37.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.289+0000 7f1e5856c640 1 -- start start 2024-09-21T09:41:37.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.289+0000 7f1e5856c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1e50094ff0 0x7f1e5008f3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:37.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.289+0000 7f1e5856c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f1e50129f90 0x7f1e5008f910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:37.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.289+0000 7f1e5856c640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1e5012a8b0 0x7f1e500720f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:37.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.289+0000 7f1e5856c640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f1e50132800 con 0x7f1e50094ff0 2024-09-21T09:41:37.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.289+0000 7f1e5856c640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f1e50132680 con 0x7f1e50129f90 2024-09-21T09:41:37.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.289+0000 7f1e5856c640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f1e50132980 con 0x7f1e5012a8b0 2024-09-21T09:41:37.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.289+0000 7f1e56308640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1e50094ff0 0x7f1e5008f3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:37.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.289+0000 7f1e56308640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1e50094ff0 0x7f1e5008f3d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:39532/0 (socket says 172.21.15.92:39532) 2024-09-21T09:41:37.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.289+0000 7f1e56308640 1 -- 172.21.15.92:0/3126053308 learned_addr learned my addr 172.21.15.92:0/3126053308 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:37.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.289+0000 7f1e56b09640 1 --2- 172.21.15.92:0/3126053308 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1e5012a8b0 0x7f1e500720f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:37.295 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.289+0000 7f1e55b07640 1 --2- 172.21.15.92:0/3126053308 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f1e50129f90 0x7f1e5008f910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:37.295 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.290+0000 7f1e56308640 1 -- 172.21.15.92:0/3126053308 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1e5012a8b0 msgr2=0x7f1e500720f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:37.295 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.290+0000 7f1e56308640 1 --2- 172.21.15.92:0/3126053308 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1e5012a8b0 0x7f1e500720f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.295 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.290+0000 7f1e56308640 1 -- 172.21.15.92:0/3126053308 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f1e50129f90 msgr2=0x7f1e5008f910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:37.295 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.290+0000 7f1e56308640 1 --2- 172.21.15.92:0/3126053308 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f1e50129f90 0x7f1e5008f910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.295 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.290+0000 7f1e56308640 1 -- 172.21.15.92:0/3126053308 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1e500910c0 con 0x7f1e50094ff0 2024-09-21T09:41:37.295 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.290+0000 7f1e56308640 1 --2- 172.21.15.92:0/3126053308 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1e50094ff0 0x7f1e5008f3d0 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f1e4c000c00 tx=0x7f1e4c032c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:37.295 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.290+0000 7f1e46ffd640 1 -- 172.21.15.92:0/3126053308 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1e4c040070 con 0x7f1e50094ff0 2024-09-21T09:41:37.295 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.290+0000 7f1e46ffd640 1 -- 172.21.15.92:0/3126053308 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f1e4c0038e0 con 0x7f1e50094ff0 2024-09-21T09:41:37.295 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.290+0000 7f1e5856c640 1 -- 172.21.15.92:0/3126053308 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1e5008fe50 con 0x7f1e50094ff0 2024-09-21T09:41:37.295 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.290+0000 7f1e46ffd640 1 -- 172.21.15.92:0/3126053308 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1e4c003d30 con 0x7f1e50094ff0 2024-09-21T09:41:37.295 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.290+0000 7f1e5856c640 1 -- 172.21.15.92:0/3126053308 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1e50091f30 con 0x7f1e50094ff0 2024-09-21T09:41:37.295 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.290+0000 7f1e46ffd640 1 -- 172.21.15.92:0/3126053308 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f1e4c039040 con 0x7f1e50094ff0 2024-09-21T09:41:37.295 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.290+0000 7f1e46ffd640 1 -- 172.21.15.92:0/3126053308 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(4..4 src has 1..4) ==== 955+0+0 (secure 0 0 0) 0x7f1e4c045d70 con 0x7f1e50094ff0 2024-09-21T09:41:37.296 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.291+0000 7f1e5856c640 1 -- 172.21.15.92:0/3126053308 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1e5012a370 con 0x7f1e50094ff0 2024-09-21T09:41:37.296 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:37.292+0000 7fdc77016040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:41:37.296 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.294+0000 7f1e46ffd640 1 -- 172.21.15.92:0/3126053308 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7f1e500910c0 con 0x7f1e50094ff0 2024-09-21T09:41:37.298 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:37.296+0000 7ffbab4a0040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:41:37.342 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:37.339+0000 7fdc77016040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:41:37.345 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:37.342+0000 7ffbab4a0040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:41:37.366 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.363+0000 7f1e5856c640 1 -- 172.21.15.92:0/3126053308 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd new", "uuid": "5c587421-3409-4469-a2ee-b2ea20cd45e5", "id": 2} v 0) -- 0x7f1e50069140 con 0x7f1e50094ff0 2024-09-21T09:41:37.367 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.364+0000 7f1e46ffd640 1 -- 172.21.15.92:0/3126053308 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "5c587421-3409-4469-a2ee-b2ea20cd45e5", "id": 2}]=0 v5) ==== 112+0+2 (secure 0 0 0) 0x7f1e4c045460 con 0x7f1e50094ff0 2024-09-21T09:41:37.367 INFO:teuthology.orchestra.run.smithi092.stdout:2 2024-09-21T09:41:37.369 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.366+0000 7f1e5856c640 1 -- 172.21.15.92:0/3126053308 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1e50094ff0 msgr2=0x7f1e5008f3d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:37.369 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.366+0000 7f1e5856c640 1 --2- 172.21.15.92:0/3126053308 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1e50094ff0 0x7f1e5008f3d0 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f1e4c000c00 tx=0x7f1e4c032c10 comp rx=0 tx=0).stop 2024-09-21T09:41:37.369 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.366+0000 7f1e5856c640 1 -- 172.21.15.92:0/3126053308 shutdown_connections 2024-09-21T09:41:37.369 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.366+0000 7f1e5856c640 1 --2- 172.21.15.92:0/3126053308 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1e5012a8b0 0x7f1e500720f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.369 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.366+0000 7f1e5856c640 1 --2- 172.21.15.92:0/3126053308 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f1e50129f90 0x7f1e5008f910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.369 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.366+0000 7f1e5856c640 1 --2- 172.21.15.92:0/3126053308 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1e50094ff0 0x7f1e5008f3d0 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.369 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.366+0000 7f1e5856c640 1 -- 172.21.15.92:0/3126053308 >> 172.21.15.92:0/3126053308 conn(0x7f1e5009ca00 msgr2=0x7f1e50121490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:37.370 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.366+0000 7f1e5856c640 1 -- 172.21.15.92:0/3126053308 shutdown_connections 2024-09-21T09:41:37.370 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.366+0000 7f1e5856c640 1 -- 172.21.15.92:0/3126053308 wait complete. 2024-09-21T09:41:37.383 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph osd new b98663b7-de4d-4c12-ac91-8c33f159da35 3 2024-09-21T09:41:37.458 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:37.455+0000 7fdc77016040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:41:37.458 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:37.456+0000 7ffbab4a0040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:41:37.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.481+0000 7f61e3b65640 1 -- 172.21.15.92:0/4157368902 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61dc09ac00 msgr2=0x7f61dc09afe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:37.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.481+0000 7f61e3b65640 1 --2- 172.21.15.92:0/4157368902 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61dc09ac00 0x7f61dc09afe0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f61d8009860 tx=0x7f61d802a420 comp rx=0 tx=0).stop 2024-09-21T09:41:37.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.481+0000 7f61e3b65640 1 -- 172.21.15.92:0/4157368902 shutdown_connections 2024-09-21T09:41:37.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.481+0000 7f61e3b65640 1 --2- 172.21.15.92:0/4157368902 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61dc09b520 0x7f61dc135ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.481+0000 7f61e3b65640 1 --2- 172.21.15.92:0/4157368902 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61dc09ac00 0x7f61dc09afe0 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.481+0000 7f61e3b65640 1 -- 172.21.15.92:0/4157368902 >> 172.21.15.92:0/4157368902 conn(0x7f61dc09e830 msgr2=0x7f61dc0a0300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:37.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.481+0000 7f61e3b65640 1 -- 172.21.15.92:0/4157368902 shutdown_connections 2024-09-21T09:41:37.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.481+0000 7f61e3b65640 1 -- 172.21.15.92:0/4157368902 wait complete. 2024-09-21T09:41:37.487 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.482+0000 7f61e3b65640 1 Processor -- start 2024-09-21T09:41:37.487 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.482+0000 7f61e3b65640 1 -- start start 2024-09-21T09:41:37.487 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.483+0000 7f61e3b65640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61dc09ac00 0x7f61dc090d00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:37.487 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.483+0000 7f61e3b65640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f61dc09b520 0x7f61dc091240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:37.487 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.483+0000 7f61e3b65640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61dc0a0bf0 0x7f61dc091780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:37.487 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.483+0000 7f61e3b65640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f61dc136b60 con 0x7f61dc0a0bf0 2024-09-21T09:41:37.487 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.483+0000 7f61e3b65640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f61dc1369e0 con 0x7f61dc09b520 2024-09-21T09:41:37.487 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.483+0000 7f61e1901640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61dc09ac00 0x7f61dc090d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:37.487 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.483+0000 7f61e1901640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61dc09ac00 0x7f61dc090d00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:34798/0 (socket says 172.21.15.92:34798) 2024-09-21T09:41:37.487 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.483+0000 7f61e2102640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61dc0a0bf0 0x7f61dc091780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:37.487 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.483+0000 7f61e1901640 1 -- 172.21.15.92:0/741400883 learned_addr learned my addr 172.21.15.92:0/741400883 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:37.487 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.483+0000 7f61e1100640 1 --2- 172.21.15.92:0/741400883 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f61dc09b520 0x7f61dc091240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:37.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.483+0000 7f61e3b65640 1 -- 172.21.15.92:0/741400883 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f61dc136ce0 con 0x7f61dc09ac00 2024-09-21T09:41:37.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.483+0000 7f61e2102640 1 -- 172.21.15.92:0/741400883 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61dc09ac00 msgr2=0x7f61dc090d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:37.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.483+0000 7f61e2102640 1 --2- 172.21.15.92:0/741400883 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61dc09ac00 0x7f61dc090d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.483+0000 7f61e2102640 1 -- 172.21.15.92:0/741400883 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f61dc09b520 msgr2=0x7f61dc091240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:37.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.483+0000 7f61e2102640 1 --2- 172.21.15.92:0/741400883 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f61dc09b520 0x7f61dc091240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.483+0000 7f61e2102640 1 -- 172.21.15.92:0/741400883 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f61dc125d10 con 0x7f61dc0a0bf0 2024-09-21T09:41:37.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.483+0000 7f61e1100640 1 --2- 172.21.15.92:0/741400883 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f61dc09b520 0x7f61dc091240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:37.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.483+0000 7f61e1901640 1 --2- 172.21.15.92:0/741400883 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61dc09ac00 0x7f61dc090d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:37.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.483+0000 7f61e2102640 1 --2- 172.21.15.92:0/741400883 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61dc0a0bf0 0x7f61dc091780 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f61d400efc0 tx=0x7f61d400c800 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:37.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.484+0000 7f61d27fc640 1 -- 172.21.15.92:0/741400883 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f61d4003970 con 0x7f61dc0a0bf0 2024-09-21T09:41:37.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.484+0000 7f61d27fc640 1 -- 172.21.15.92:0/741400883 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f61d4003c00 con 0x7f61dc0a0bf0 2024-09-21T09:41:37.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.484+0000 7f61e3b65640 1 -- 172.21.15.92:0/741400883 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f61dc1265e0 con 0x7f61dc0a0bf0 2024-09-21T09:41:37.489 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.484+0000 7f61d27fc640 1 -- 172.21.15.92:0/741400883 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f61d4004020 con 0x7f61dc0a0bf0 2024-09-21T09:41:37.489 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.484+0000 7f61e3b65640 1 -- 172.21.15.92:0/741400883 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f61dc091cc0 con 0x7f61dc0a0bf0 2024-09-21T09:41:37.489 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.484+0000 7f61d27fc640 1 -- 172.21.15.92:0/741400883 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f61d401d020 con 0x7f61dc0a0bf0 2024-09-21T09:41:37.489 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.484+0000 7f61d27fc640 1 -- 172.21.15.92:0/741400883 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(5..5 src has 1..5) ==== 1070+0+0 (secure 0 0 0) 0x7f61d400bbd0 con 0x7f61dc0a0bf0 2024-09-21T09:41:37.489 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.485+0000 7f61e3b65640 1 -- 172.21.15.92:0/741400883 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f61dc09afe0 con 0x7f61dc0a0bf0 2024-09-21T09:41:37.489 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.487+0000 7f61d27fc640 1 -- 172.21.15.92:0/741400883 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7f61d4024070 con 0x7f61dc0a0bf0 2024-09-21T09:41:37.512 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:37.509+0000 7ffbab4a0040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:41:37.523 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:37.521+0000 7fdc77016040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:41:37.565 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:37.562+0000 7ffbab4a0040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:41:37.588 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:37.586+0000 7fdc77016040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:41:37.599 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.596+0000 7f61e3b65640 1 -- 172.21.15.92:0/741400883 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd new", "uuid": "b98663b7-de4d-4c12-ac91-8c33f159da35", "id": 3} v 0) -- 0x7f61dc02d7c0 con 0x7f61dc0a0bf0 2024-09-21T09:41:37.600 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.598+0000 7f61d27fc640 1 -- 172.21.15.92:0/741400883 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "b98663b7-de4d-4c12-ac91-8c33f159da35", "id": 3}]=0 v6) ==== 112+0+2 (secure 0 0 0) 0x7f61d400b630 con 0x7f61dc0a0bf0 2024-09-21T09:41:37.600 INFO:teuthology.orchestra.run.smithi092.stdout:3 2024-09-21T09:41:37.602 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.599+0000 7f61e3b65640 1 -- 172.21.15.92:0/741400883 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61dc0a0bf0 msgr2=0x7f61dc091780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:37.602 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.599+0000 7f61e3b65640 1 --2- 172.21.15.92:0/741400883 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61dc0a0bf0 0x7f61dc091780 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f61d400efc0 tx=0x7f61d400c800 comp rx=0 tx=0).stop 2024-09-21T09:41:37.602 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.599+0000 7f61e3b65640 1 -- 172.21.15.92:0/741400883 shutdown_connections 2024-09-21T09:41:37.602 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.599+0000 7f61e3b65640 1 --2- 172.21.15.92:0/741400883 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61dc0a0bf0 0x7f61dc091780 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.602 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.599+0000 7f61e3b65640 1 --2- 172.21.15.92:0/741400883 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f61dc09b520 0x7f61dc091240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.602 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.599+0000 7f61e3b65640 1 --2- 172.21.15.92:0/741400883 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61dc09ac00 0x7f61dc090d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.602 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.599+0000 7f61e3b65640 1 -- 172.21.15.92:0/741400883 >> 172.21.15.92:0/741400883 conn(0x7f61dc09e830 msgr2=0x7f61dc135950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:37.603 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.599+0000 7f61e3b65640 1 -- 172.21.15.92:0/741400883 shutdown_connections 2024-09-21T09:41:37.603 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.599+0000 7f61e3b65640 1 -- 172.21.15.92:0/741400883 wait complete. 2024-09-21T09:41:37.612 INFO:tasks.ceph.osd.0:Restarting daemon 2024-09-21T09:41:37.613 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 0 2024-09-21T09:41:37.615 INFO:tasks.ceph.osd.0:Started 2024-09-21T09:41:37.615 INFO:tasks.ceph.osd.1:Restarting daemon 2024-09-21T09:41:37.615 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 1 2024-09-21T09:41:37.617 INFO:tasks.ceph.osd.1:Started 2024-09-21T09:41:37.617 INFO:tasks.ceph.osd.2:Restarting daemon 2024-09-21T09:41:37.617 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 2 2024-09-21T09:41:37.619 INFO:tasks.ceph.osd.2:Started 2024-09-21T09:41:37.619 INFO:tasks.ceph.osd.3:Restarting daemon 2024-09-21T09:41:37.619 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 3 2024-09-21T09:41:37.622 INFO:tasks.ceph.osd.3:Started 2024-09-21T09:41:37.622 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:41:37.648 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:37.645+0000 7ffbab4a0040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:41:37.660 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:37.658+0000 7fdc77016040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:41:37.718 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:37.715+0000 7ffbab4a0040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:41:37.720 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:37.718+0000 7fdc77016040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:41:37.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.758+0000 7f85347a5640 1 -- 172.21.15.92:0/766012260 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f852c130f60 msgr2=0x7f852c133350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:37.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.758+0000 7f85347a5640 1 --2- 172.21.15.92:0/766012260 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f852c130f60 0x7f852c133350 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f851c009900 tx=0x7f851c02a530 comp rx=0 tx=0).stop 2024-09-21T09:41:37.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.759+0000 7f85347a5640 1 -- 172.21.15.92:0/766012260 shutdown_connections 2024-09-21T09:41:37.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.759+0000 7f85347a5640 1 --2- 172.21.15.92:0/766012260 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f852c133890 0x7f852c135c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.762 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.759+0000 7f85347a5640 1 --2- 172.21.15.92:0/766012260 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f852c130f60 0x7f852c133350 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.762 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.759+0000 7f85347a5640 1 -- 172.21.15.92:0/766012260 >> 172.21.15.92:0/766012260 conn(0x7f852c0a0a40 msgr2=0x7f852c09cf10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:37.762 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.759+0000 7f85347a5640 1 -- 172.21.15.92:0/766012260 shutdown_connections 2024-09-21T09:41:37.762 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.759+0000 7f85347a5640 1 -- 172.21.15.92:0/766012260 wait complete. 2024-09-21T09:41:37.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.759+0000 7f85347a5640 1 Processor -- start 2024-09-21T09:41:37.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.759+0000 7f85347a5640 1 -- start start 2024-09-21T09:41:37.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.760+0000 7f85347a5640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f852c06eb50 0x7f852c094610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:37.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.760+0000 7f85347a5640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f852c130f60 0x7f852c094b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:37.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.760+0000 7f85347a5640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f852c133890 0x7f852c095090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:37.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.760+0000 7f85347a5640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f852c136bd0 con 0x7f852c130f60 2024-09-21T09:41:37.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.760+0000 7f85347a5640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f852c136a50 con 0x7f852c133890 2024-09-21T09:41:37.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.760+0000 7f85347a5640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f852c136d50 con 0x7f852c06eb50 2024-09-21T09:41:37.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.760+0000 7f8532541640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f852c06eb50 0x7f852c094610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:37.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.760+0000 7f8531d40640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f852c130f60 0x7f852c094b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:37.764 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.760+0000 7f8532541640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f852c06eb50 0x7f852c094610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:34836/0 (socket says 172.21.15.92:34836) 2024-09-21T09:41:37.764 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.760+0000 7f8532541640 1 -- 172.21.15.92:0/4175075553 learned_addr learned my addr 172.21.15.92:0/4175075553 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:37.764 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.760+0000 7f8532d42640 1 --2- 172.21.15.92:0/4175075553 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f852c133890 0x7f852c095090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:37.764 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.760+0000 7f8532541640 1 -- 172.21.15.92:0/4175075553 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f852c133890 msgr2=0x7f852c095090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:37.764 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.760+0000 7f8532541640 1 --2- 172.21.15.92:0/4175075553 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f852c133890 0x7f852c095090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.765 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.760+0000 7f8532541640 1 -- 172.21.15.92:0/4175075553 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f852c130f60 msgr2=0x7f852c094b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:37.765 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.760+0000 7f8532541640 1 --2- 172.21.15.92:0/4175075553 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f852c130f60 0x7f852c094b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.765 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.760+0000 7f8532541640 1 -- 172.21.15.92:0/4175075553 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f852c12ba30 con 0x7f852c06eb50 2024-09-21T09:41:37.765 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.761+0000 7f8532541640 1 --2- 172.21.15.92:0/4175075553 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f852c06eb50 0x7f852c094610 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f852800c460 tx=0x7f852800c920 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:37.765 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.761+0000 7f8522ffd640 1 -- 172.21.15.92:0/4175075553 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8528005480 con 0x7f852c06eb50 2024-09-21T09:41:37.765 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.761+0000 7f8522ffd640 1 -- 172.21.15.92:0/4175075553 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f85280035f0 con 0x7f852c06eb50 2024-09-21T09:41:37.765 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.761+0000 7f85347a5640 1 -- 172.21.15.92:0/4175075553 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f852c096ba0 con 0x7f852c06eb50 2024-09-21T09:41:37.765 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.761+0000 7f8522ffd640 1 -- 172.21.15.92:0/4175075553 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8528003a60 con 0x7f852c06eb50 2024-09-21T09:41:37.765 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.761+0000 7f85347a5640 1 -- 172.21.15.92:0/4175075553 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f852c1c9fe0 con 0x7f852c06eb50 2024-09-21T09:41:37.765 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.761+0000 7f8522ffd640 1 -- 172.21.15.92:0/4175075553 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 1) ==== 807+0+0 (secure 0 0 0) 0x7f8528003c00 con 0x7f852c06eb50 2024-09-21T09:41:37.765 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.761+0000 7f8522ffd640 1 -- 172.21.15.92:0/4175075553 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(6..6 src has 1..6) ==== 1185+0+0 (secure 0 0 0) 0x7f852801b910 con 0x7f852c06eb50 2024-09-21T09:41:37.766 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.762+0000 7f85347a5640 1 -- 172.21.15.92:0/4175075553 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f852c096ee0 con 0x7f852c06eb50 2024-09-21T09:41:37.767 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.764+0000 7f8522ffd640 1 -- 172.21.15.92:0/4175075553 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7f852c096ee0 con 0x7f852c06eb50 2024-09-21T09:41:37.769 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:37.767+0000 7ffbab4a0040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:41:37.772 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:37.769+0000 7fdc77016040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:41:37.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.822+0000 7f85347a5640 1 -- 172.21.15.92:0/4175075553 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f852c06ef30 con 0x7f852c06eb50 2024-09-21T09:41:37.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.823+0000 7f8522ffd640 1 -- 172.21.15.92:0/4175075553 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v6) ==== 74+0+4153 (secure 0 0 0) 0x7f852801bc10 con 0x7f852c06eb50 2024-09-21T09:41:37.826 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:37.826 INFO:teuthology.orchestra.run.smithi092.stdout:{"epoch":6,"fsid":"5f250e6c-4e36-49b1-ab62-d9454369fa03","created":"2024-09-21T09:41:34.680338+0000","modified":"2024-09-21T09:41:37.598778+0000","last_up_change":"0.000000","last_in_change":"2024-09-21T09:41:37.598778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"f3314f59-3374-4072-85c9-f8c4e458efc8","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"cf0486af-ba68-4763-a364-c1f2181604a7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"5c587421-3409-4469-a2ee-b2ea20cd45e5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"b98663b7-de4d-4c12-ac91-8c33f159da35","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:41:37.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.824+0000 7f85347a5640 1 -- 172.21.15.92:0/4175075553 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f852c06eb50 msgr2=0x7f852c094610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:37.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.824+0000 7f85347a5640 1 --2- 172.21.15.92:0/4175075553 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f852c06eb50 0x7f852c094610 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f852800c460 tx=0x7f852800c920 comp rx=0 tx=0).stop 2024-09-21T09:41:37.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.824+0000 7f85347a5640 1 -- 172.21.15.92:0/4175075553 shutdown_connections 2024-09-21T09:41:37.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.824+0000 7f85347a5640 1 --2- 172.21.15.92:0/4175075553 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f852c133890 0x7f852c095090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.824+0000 7f85347a5640 1 --2- 172.21.15.92:0/4175075553 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f852c130f60 0x7f852c094b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.824+0000 7f85347a5640 1 --2- 172.21.15.92:0/4175075553 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f852c06eb50 0x7f852c094610 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:37.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.824+0000 7f85347a5640 1 -- 172.21.15.92:0/4175075553 >> 172.21.15.92:0/4175075553 conn(0x7f852c0a0a40 msgr2=0x7f852c133c70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:37.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.824+0000 7f85347a5640 1 -- 172.21.15.92:0/4175075553 shutdown_connections 2024-09-21T09:41:37.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:37.824+0000 7f85347a5640 1 -- 172.21.15.92:0/4175075553 wait complete. 2024-09-21T09:41:37.837 INFO:tasks.ceph.ceph_manager.ceph:[] 2024-09-21T09:41:37.837 INFO:tasks.ceph:Waiting for OSDs to come up 2024-09-21T09:41:38.005 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:38.002+0000 7fdc77016040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:41:38.006 INFO:tasks.ceph.mgr.x.smithi092.stderr:2024-09-21T09:41:38.003+0000 7ffbab4a0040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:41:38.239 DEBUG:teuthology.orchestra.run.smithi092:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-09-21T09:41:38.331 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.328+0000 7ff2ff612640 1 -- 172.21.15.92:0/3892119850 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff2f8095490 msgr2=0x7ff2f8095870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:38.331 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.328+0000 7ff2ff612640 1 --2- 172.21.15.92:0/3892119850 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff2f8095490 0x7ff2f8095870 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7ff2f4009830 tx=0x7ff2f402a360 comp rx=0 tx=0).stop 2024-09-21T09:41:38.331 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.328+0000 7ff2ff612640 1 -- 172.21.15.92:0/3892119850 shutdown_connections 2024-09-21T09:41:38.331 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.328+0000 7ff2ff612640 1 --2- 172.21.15.92:0/3892119850 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff2f812af60 0x7ff2f812b410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:38.331 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.328+0000 7ff2ff612640 1 --2- 172.21.15.92:0/3892119850 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff2f806c6a0 0x7ff2f8095db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:38.331 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.328+0000 7ff2ff612640 1 --2- 172.21.15.92:0/3892119850 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff2f8095490 0x7ff2f8095870 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:38.332 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.328+0000 7ff2ff612640 1 -- 172.21.15.92:0/3892119850 >> 172.21.15.92:0/3892119850 conn(0x7ff2f809cff0 msgr2=0x7ff2f809d400 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:38.332 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.328+0000 7ff2ff612640 1 -- 172.21.15.92:0/3892119850 shutdown_connections 2024-09-21T09:41:38.332 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.328+0000 7ff2ff612640 1 -- 172.21.15.92:0/3892119850 wait complete. 2024-09-21T09:41:38.333 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.329+0000 7ff2ff612640 1 Processor -- start 2024-09-21T09:41:38.334 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.329+0000 7ff2ff612640 1 -- start start 2024-09-21T09:41:38.334 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.329+0000 7ff2ff612640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff2f806c6a0 0x7ff2f81c0fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:38.334 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.329+0000 7ff2ff612640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff2f8095490 0x7ff2f81c1500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:38.334 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.329+0000 7ff2ff612640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff2f812af60 0x7ff2f808b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:38.334 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.329+0000 7ff2ff612640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7ff2f81328f0 con 0x7ff2f806c6a0 2024-09-21T09:41:38.334 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.330+0000 7ff2ff612640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7ff2f8132770 con 0x7ff2f812af60 2024-09-21T09:41:38.334 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.330+0000 7ff2ff612640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7ff2f8132a70 con 0x7ff2f8095490 2024-09-21T09:41:38.334 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.330+0000 7ff2fd3ae640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff2f806c6a0 0x7ff2f81c0fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:38.334 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.330+0000 7ff2fd3ae640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff2f806c6a0 0x7ff2f81c0fc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:39704/0 (socket says 172.21.15.92:39704) 2024-09-21T09:41:38.334 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.330+0000 7ff2fd3ae640 1 -- 172.21.15.92:0/2177802755 learned_addr learned my addr 172.21.15.92:0/2177802755 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:38.334 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.330+0000 7ff2fcbad640 1 --2- 172.21.15.92:0/2177802755 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff2f8095490 0x7ff2f81c1500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:38.334 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.330+0000 7ff2fdbaf640 1 --2- 172.21.15.92:0/2177802755 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff2f812af60 0x7ff2f808b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:38.334 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.330+0000 7ff2fd3ae640 1 -- 172.21.15.92:0/2177802755 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff2f8095490 msgr2=0x7ff2f81c1500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:38.334 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.330+0000 7ff2fd3ae640 1 --2- 172.21.15.92:0/2177802755 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff2f8095490 0x7ff2f81c1500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:38.335 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.330+0000 7ff2fd3ae640 1 -- 172.21.15.92:0/2177802755 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff2f812af60 msgr2=0x7ff2f808b9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:38.335 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.330+0000 7ff2fd3ae640 1 --2- 172.21.15.92:0/2177802755 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff2f812af60 0x7ff2f808b9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:38.335 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.330+0000 7ff2fd3ae640 1 -- 172.21.15.92:0/2177802755 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff2f80734a0 con 0x7ff2f806c6a0 2024-09-21T09:41:38.335 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.330+0000 7ff2fd3ae640 1 --2- 172.21.15.92:0/2177802755 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff2f806c6a0 0x7ff2f81c0fc0 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7ff2f4009780 tx=0x7ff2f4033b70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:38.335 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.330+0000 7ff2edffb640 1 -- 172.21.15.92:0/2177802755 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff2f403f020 con 0x7ff2f806c6a0 2024-09-21T09:41:38.335 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.330+0000 7ff2edffb640 1 -- 172.21.15.92:0/2177802755 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7ff2f40044c0 con 0x7ff2f806c6a0 2024-09-21T09:41:38.335 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.330+0000 7ff2ff612640 1 -- 172.21.15.92:0/2177802755 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff2f81c1a40 con 0x7ff2f806c6a0 2024-09-21T09:41:38.335 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.330+0000 7ff2edffb640 1 -- 172.21.15.92:0/2177802755 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff2f4037040 con 0x7ff2f806c6a0 2024-09-21T09:41:38.335 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.330+0000 7ff2ff612640 1 -- 172.21.15.92:0/2177802755 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff2f808e5f0 con 0x7ff2f806c6a0 2024-09-21T09:41:38.335 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.331+0000 7ff2edffb640 1 -- 172.21.15.92:0/2177802755 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 2) ==== 28070+0+0 (secure 0 0 0) 0x7ff2f4042020 con 0x7ff2f806c6a0 2024-09-21T09:41:38.335 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.331+0000 7ff2edffb640 1 -- 172.21.15.92:0/2177802755 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(6..6 src has 1..6) ==== 1185+0+0 (secure 0 0 0) 0x7ff2f4054010 con 0x7ff2f806c6a0 2024-09-21T09:41:38.336 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.331+0000 7ff2ff612640 1 -- 172.21.15.92:0/2177802755 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff2f812b410 con 0x7ff2f806c6a0 2024-09-21T09:41:38.336 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.333+0000 7ff2edffb640 1 -- 172.21.15.92:0/2177802755 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+74432 (secure 0 0 0) 0x7ff2f4049030 con 0x7ff2f806c6a0 2024-09-21T09:41:38.392 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.390+0000 7ff2ff612640 1 -- 172.21.15.92:0/2177802755 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7ff2f808e8e0 con 0x7ff2f806c6a0 2024-09-21T09:41:38.393 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.390+0000 7ff2edffb640 1 -- 172.21.15.92:0/2177802755 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v6) ==== 74+0+4153 (secure 0 0 0) 0x7ff2f403a510 con 0x7ff2f806c6a0 2024-09-21T09:41:38.393 INFO:teuthology.misc.health.smithi092.stdout: 2024-09-21T09:41:38.393 INFO:teuthology.misc.health.smithi092.stdout:{"epoch":6,"fsid":"5f250e6c-4e36-49b1-ab62-d9454369fa03","created":"2024-09-21T09:41:34.680338+0000","modified":"2024-09-21T09:41:37.598778+0000","last_up_change":"0.000000","last_in_change":"2024-09-21T09:41:37.598778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"f3314f59-3374-4072-85c9-f8c4e458efc8","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"cf0486af-ba68-4763-a364-c1f2181604a7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"5c587421-3409-4469-a2ee-b2ea20cd45e5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"b98663b7-de4d-4c12-ac91-8c33f159da35","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:41:38.394 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.391+0000 7ff2ff612640 1 -- 172.21.15.92:0/2177802755 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff2f806c6a0 msgr2=0x7ff2f81c0fc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:38.394 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.391+0000 7ff2ff612640 1 --2- 172.21.15.92:0/2177802755 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff2f806c6a0 0x7ff2f81c0fc0 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7ff2f4009780 tx=0x7ff2f4033b70 comp rx=0 tx=0).stop 2024-09-21T09:41:38.394 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.391+0000 7ff2ff612640 1 -- 172.21.15.92:0/2177802755 shutdown_connections 2024-09-21T09:41:38.394 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.391+0000 7ff2ff612640 1 --2- 172.21.15.92:0/2177802755 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff2f812af60 0x7ff2f808b9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:38.395 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.391+0000 7ff2ff612640 1 --2- 172.21.15.92:0/2177802755 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff2f8095490 0x7ff2f81c1500 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:38.395 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.391+0000 7ff2ff612640 1 --2- 172.21.15.92:0/2177802755 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff2f806c6a0 0x7ff2f81c0fc0 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:38.395 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.391+0000 7ff2ff612640 1 -- 172.21.15.92:0/2177802755 >> 172.21.15.92:0/2177802755 conn(0x7ff2f809cff0 msgr2=0x7ff2f8099400 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:38.395 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.391+0000 7ff2ff612640 1 -- 172.21.15.92:0/2177802755 shutdown_connections 2024-09-21T09:41:38.395 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:38.391+0000 7ff2ff612640 1 -- 172.21.15.92:0/2177802755 wait complete. 2024-09-21T09:41:38.403 DEBUG:teuthology.misc:0 of 4 OSDs are up 2024-09-21T09:41:39.015 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:39.013+0000 7fdc5fb5f640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-21T09:41:39.017 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:41:39.015+0000 7fdc5fb5f640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-21T09:41:39.128 INFO:tasks.ceph.osd.3.smithi092.stderr:2024-09-21T09:41:39.125+0000 7f4b6cc0d640 -1 Falling back to public interface 2024-09-21T09:41:39.351 INFO:tasks.ceph.osd.0.smithi092.stderr:2024-09-21T09:41:39.348+0000 7febd20b2640 -1 Falling back to public interface 2024-09-21T09:41:39.587 INFO:tasks.ceph.osd.2.smithi092.stderr:2024-09-21T09:41:39.584+0000 7fd0631dd640 -1 Falling back to public interface 2024-09-21T09:41:39.592 INFO:tasks.ceph.osd.1.smithi092.stderr:2024-09-21T09:41:39.589+0000 7f4c54b20640 -1 Falling back to public interface 2024-09-21T09:41:43.666 INFO:tasks.ceph.osd.3.smithi092.stderr:2024-09-21T09:41:43.664+0000 7f4b6cc0d640 -1 osd.3 0 log_to_monitors true 2024-09-21T09:41:43.882 INFO:tasks.ceph.osd.2.smithi092.stderr:2024-09-21T09:41:43.880+0000 7fd0631dd640 -1 osd.2 0 log_to_monitors true 2024-09-21T09:41:43.906 INFO:tasks.ceph.osd.0.smithi092.stderr:2024-09-21T09:41:43.904+0000 7febd20b2640 -1 osd.0 0 log_to_monitors true 2024-09-21T09:41:44.117 INFO:tasks.ceph.osd.1.smithi092.stderr:2024-09-21T09:41:44.115+0000 7f4c54b20640 -1 osd.1 0 log_to_monitors true 2024-09-21T09:41:44.806 DEBUG:teuthology.orchestra.run.smithi092:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-09-21T09:41:44.909 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.907+0000 7f06287d4640 1 -- 172.21.15.92:0/3539080055 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0620072d40 msgr2=0x7f062012bb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:44.910 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.907+0000 7f06287d4640 1 --2- 172.21.15.92:0/3539080055 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0620072d40 0x7f062012bb00 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f0610009830 tx=0x7f061002a360 comp rx=0 tx=0).stop 2024-09-21T09:41:44.910 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.907+0000 7f06287d4640 1 -- 172.21.15.92:0/3539080055 shutdown_connections 2024-09-21T09:41:44.910 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.907+0000 7f06287d4640 1 --2- 172.21.15.92:0/3539080055 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f062012c040 0x7f062012c4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:44.910 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.907+0000 7f06287d4640 1 --2- 172.21.15.92:0/3539080055 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0620072d40 0x7f062012bb00 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:44.910 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.908+0000 7f06287d4640 1 --2- 172.21.15.92:0/3539080055 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0620128b30 0x7f0620128f10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:44.910 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.908+0000 7f06287d4640 1 -- 172.21.15.92:0/3539080055 >> 172.21.15.92:0/3539080055 conn(0x7f062009b5e0 msgr2=0x7f062009b9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:44.910 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.908+0000 7f06287d4640 1 -- 172.21.15.92:0/3539080055 shutdown_connections 2024-09-21T09:41:44.910 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.908+0000 7f06287d4640 1 -- 172.21.15.92:0/3539080055 wait complete. 2024-09-21T09:41:44.912 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.909+0000 7f06287d4640 1 Processor -- start 2024-09-21T09:41:44.912 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.909+0000 7f06287d4640 1 -- start start 2024-09-21T09:41:44.912 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.909+0000 7f06287d4640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0620072d40 0x7f06201215d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:44.912 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.909+0000 7f06287d4640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0620128b30 0x7f0620121b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:44.912 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.909+0000 7f06287d4640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f062012c040 0x7f062012fab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:44.912 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.909+0000 7f06287d4640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f06201336e0 con 0x7f0620128b30 2024-09-21T09:41:44.912 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.909+0000 7f06287d4640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f0620133560 con 0x7f0620072d40 2024-09-21T09:41:44.912 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.909+0000 7f06287d4640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f0620133860 con 0x7f062012c040 2024-09-21T09:41:44.912 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.909+0000 7f0626570640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0620072d40 0x7f06201215d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:44.912 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.909+0000 7f0625d6f640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0620128b30 0x7f0620121b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:44.912 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.909+0000 7f0626570640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0620072d40 0x7f06201215d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:56096/0 (socket says 172.21.15.92:56096) 2024-09-21T09:41:44.913 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.909+0000 7f0625d6f640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0620128b30 0x7f0620121b10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:39730/0 (socket says 172.21.15.92:39730) 2024-09-21T09:41:44.913 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.909+0000 7f0626d71640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f062012c040 0x7f062012fab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:44.913 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.909+0000 7f0626570640 1 -- 172.21.15.92:0/1452209388 learned_addr learned my addr 172.21.15.92:0/1452209388 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:44.913 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.910+0000 7f0626570640 1 -- 172.21.15.92:0/1452209388 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f062012c040 msgr2=0x7f062012fab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:44.913 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.910+0000 7f0626570640 1 --2- 172.21.15.92:0/1452209388 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f062012c040 0x7f062012fab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:44.913 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.910+0000 7f0626570640 1 -- 172.21.15.92:0/1452209388 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0620128b30 msgr2=0x7f0620121b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:44.913 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.910+0000 7f0626570640 1 --2- 172.21.15.92:0/1452209388 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0620128b30 0x7f0620121b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:44.913 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.910+0000 7f0626570640 1 -- 172.21.15.92:0/1452209388 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0620131b20 con 0x7f0620072d40 2024-09-21T09:41:44.913 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.910+0000 7f0626570640 1 --2- 172.21.15.92:0/1452209388 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0620072d40 0x7f06201215d0 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f061c00e9f0 tx=0x7f061c00eeb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:44.913 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.910+0000 7f0616ffd640 1 -- 172.21.15.92:0/1452209388 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f061c00cd80 con 0x7f0620072d40 2024-09-21T09:41:44.913 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.910+0000 7f0616ffd640 1 -- 172.21.15.92:0/1452209388 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f061c010430 con 0x7f0620072d40 2024-09-21T09:41:44.913 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.910+0000 7f0616ffd640 1 -- 172.21.15.92:0/1452209388 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f061c010840 con 0x7f0620072d40 2024-09-21T09:41:44.913 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.910+0000 7f06287d4640 1 -- 172.21.15.92:0/1452209388 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f06201262d0 con 0x7f0620072d40 2024-09-21T09:41:44.914 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.911+0000 7f06287d4640 1 -- 172.21.15.92:0/1452209388 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f0620122050 con 0x7f0620072d40 2024-09-21T09:41:44.914 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.911+0000 7f06287d4640 1 -- 172.21.15.92:0/1452209388 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0620128f10 con 0x7f0620072d40 2024-09-21T09:41:44.914 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.912+0000 7f0616ffd640 1 -- 172.21.15.92:0/1452209388 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 4) ==== 55943+0+0 (secure 0 0 0) 0x7f061c0109e0 con 0x7f0620072d40 2024-09-21T09:41:44.914 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.912+0000 7f0616ffd640 1 -- 172.21.15.92:0/1452209388 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(8..8 src has 1..8) ==== 1465+0+0 (secure 0 0 0) 0x7f061c05def0 con 0x7f0620072d40 2024-09-21T09:41:44.918 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:44.916+0000 7f0616ffd640 1 -- 172.21.15.92:0/1452209388 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f06201262d0 con 0x7f0620072d40 2024-09-21T09:41:45.078 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:45.076+0000 7f06287d4640 1 -- 172.21.15.92:0/1452209388 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f06200694d0 con 0x7f0620072d40 2024-09-21T09:41:45.079 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:45.077+0000 7f0616ffd640 1 -- 172.21.15.92:0/1452209388 <== mon.1 v2:172.21.15.92:3301/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v8) ==== 74+0+4153 (secure 0 0 0) 0x7f061c03d830 con 0x7f0620072d40 2024-09-21T09:41:45.079 INFO:teuthology.misc.health.smithi092.stdout: 2024-09-21T09:41:45.079 INFO:teuthology.misc.health.smithi092.stdout:{"epoch":8,"fsid":"5f250e6c-4e36-49b1-ab62-d9454369fa03","created":"2024-09-21T09:41:34.680338+0000","modified":"2024-09-21T09:41:44.728463+0000","last_up_change":"0.000000","last_in_change":"2024-09-21T09:41:37.598778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"f3314f59-3374-4072-85c9-f8c4e458efc8","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"cf0486af-ba68-4763-a364-c1f2181604a7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"5c587421-3409-4469-a2ee-b2ea20cd45e5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"b98663b7-de4d-4c12-ac91-8c33f159da35","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:41:45.081 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:45.079+0000 7f06287d4640 1 -- 172.21.15.92:0/1452209388 >> v1:172.21.15.92:6800/1280404317 conn(0x7f0604047f40 legacy=0x7f060404a400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:45.082 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:45.079+0000 7f06287d4640 1 -- 172.21.15.92:0/1452209388 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0620072d40 msgr2=0x7f06201215d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:45.082 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:45.079+0000 7f06287d4640 1 --2- 172.21.15.92:0/1452209388 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0620072d40 0x7f06201215d0 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f061c00e9f0 tx=0x7f061c00eeb0 comp rx=0 tx=0).stop 2024-09-21T09:41:45.082 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:45.079+0000 7f06287d4640 1 -- 172.21.15.92:0/1452209388 shutdown_connections 2024-09-21T09:41:45.082 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:45.079+0000 7f06287d4640 1 --2- 172.21.15.92:0/1452209388 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f062012c040 0x7f062012fab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:45.082 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:45.079+0000 7f06287d4640 1 --2- 172.21.15.92:0/1452209388 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0620128b30 0x7f0620121b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:45.082 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:45.079+0000 7f06287d4640 1 --2- 172.21.15.92:0/1452209388 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0620072d40 0x7f06201215d0 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:45.082 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:45.079+0000 7f06287d4640 1 -- 172.21.15.92:0/1452209388 >> 172.21.15.92:0/1452209388 conn(0x7f062009b5e0 msgr2=0x7f06201264e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:45.082 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:45.079+0000 7f06287d4640 1 -- 172.21.15.92:0/1452209388 shutdown_connections 2024-09-21T09:41:45.082 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:45.079+0000 7f06287d4640 1 -- 172.21.15.92:0/1452209388 wait complete. 2024-09-21T09:41:45.091 DEBUG:teuthology.misc:0 of 4 OSDs are up 2024-09-21T09:41:45.632 INFO:tasks.ceph.osd.3.smithi092.stderr:2024-09-21T09:41:45.631+0000 7f4b68da7640 -1 osd.3 0 waiting for initial osdmap 2024-09-21T09:41:45.638 INFO:tasks.ceph.osd.3.smithi092.stderr:2024-09-21T09:41:45.636+0000 7f4b603a2640 -1 osd.3 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:41:47.409 INFO:tasks.ceph.osd.1.smithi092.stderr:2024-09-21T09:41:47.408+0000 7f4c5028d640 -1 osd.1 0 waiting for initial osdmap 2024-09-21T09:41:47.416 INFO:tasks.ceph.osd.1.smithi092.stderr:2024-09-21T09:41:47.414+0000 7f4c488bf640 -1 osd.1 10 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:41:47.424 INFO:tasks.ceph.osd.0.smithi092.stderr:2024-09-21T09:41:47.422+0000 7febcd81f640 -1 osd.0 0 waiting for initial osdmap 2024-09-21T09:41:47.427 INFO:tasks.ceph.osd.0.smithi092.stderr:2024-09-21T09:41:47.426+0000 7febc5e51640 -1 osd.0 10 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:41:47.457 INFO:tasks.ceph.osd.2.smithi092.stderr:2024-09-21T09:41:47.455+0000 7fd05e94a640 -1 osd.2 0 waiting for initial osdmap 2024-09-21T09:41:47.460 INFO:tasks.ceph.osd.2.smithi092.stderr:2024-09-21T09:41:47.458+0000 7fd056f7c640 -1 osd.2 10 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:41:51.494 DEBUG:teuthology.orchestra.run.smithi092:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-09-21T09:41:51.593 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.591+0000 7f5b7939c640 1 -- 172.21.15.92:0/4215616807 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5b7406f9a0 msgr2=0x7f5b74135970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:51.593 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.591+0000 7f5b7939c640 1 --2- 172.21.15.92:0/4215616807 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5b7406f9a0 0x7f5b74135970 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f5b6c009ca0 tx=0x7f5b6c02a9a0 comp rx=0 tx=0).stop 2024-09-21T09:41:51.594 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.591+0000 7f5b7939c640 1 -- 172.21.15.92:0/4215616807 shutdown_connections 2024-09-21T09:41:51.594 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.591+0000 7f5b7939c640 1 --2- 172.21.15.92:0/4215616807 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5b7406f9a0 0x7f5b74135970 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:51.594 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.591+0000 7f5b7939c640 1 --2- 172.21.15.92:0/4215616807 >> v2:172.21.15.92:3300/0 conn(0x7f5b7406e970 0x7f5b7406ed50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:51.594 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.591+0000 7f5b7939c640 1 -- 172.21.15.92:0/4215616807 >> 172.21.15.92:0/4215616807 conn(0x7f5b7409c6c0 msgr2=0x7f5b7409cad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:51.594 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.591+0000 7f5b7939c640 1 -- 172.21.15.92:0/4215616807 shutdown_connections 2024-09-21T09:41:51.594 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.591+0000 7f5b7939c640 1 -- 172.21.15.92:0/4215616807 wait complete. 2024-09-21T09:41:51.596 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.592+0000 7f5b7939c640 1 Processor -- start 2024-09-21T09:41:51.596 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.592+0000 7f5b7939c640 1 -- start start 2024-09-21T09:41:51.597 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.593+0000 7f5b7939c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5b7406e970 0x7f5b741c4050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:51.597 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.593+0000 7f5b7939c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5b7406f9a0 0x7f5b741c4590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:51.597 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.593+0000 7f5b7939c640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5b74070280 0x7f5b741c4ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:51.597 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.593+0000 7f5b7939c640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f5b741369d0 con 0x7f5b7406e970 2024-09-21T09:41:51.597 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.593+0000 7f5b7939c640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f5b74136850 con 0x7f5b7406f9a0 2024-09-21T09:41:51.597 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.593+0000 7f5b7939c640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f5b74136b50 con 0x7f5b74070280 2024-09-21T09:41:51.597 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.593+0000 7f5b72ffd640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5b7406e970 0x7f5b741c4050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:51.597 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.593+0000 7f5b727fc640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5b7406f9a0 0x7f5b741c4590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:51.597 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.593+0000 7f5b72ffd640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5b7406e970 0x7f5b741c4050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:54910/0 (socket says 172.21.15.92:54910) 2024-09-21T09:41:51.597 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.593+0000 7f5b72ffd640 1 -- 172.21.15.92:0/2990099673 learned_addr learned my addr 172.21.15.92:0/2990099673 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:51.597 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.593+0000 7f5b737fe640 1 --2- 172.21.15.92:0/2990099673 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5b74070280 0x7f5b741c4ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:51.598 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.593+0000 7f5b727fc640 1 -- 172.21.15.92:0/2990099673 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5b74070280 msgr2=0x7f5b741c4ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:51.598 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.593+0000 7f5b727fc640 1 --2- 172.21.15.92:0/2990099673 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5b74070280 0x7f5b741c4ad0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:51.598 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.593+0000 7f5b727fc640 1 -- 172.21.15.92:0/2990099673 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5b7406e970 msgr2=0x7f5b741c4050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:51.598 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.593+0000 7f5b727fc640 1 --2- 172.21.15.92:0/2990099673 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5b7406e970 0x7f5b741c4050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:51.598 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.593+0000 7f5b727fc640 1 -- 172.21.15.92:0/2990099673 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5b741252a0 con 0x7f5b7406f9a0 2024-09-21T09:41:51.598 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.593+0000 7f5b72ffd640 1 --2- 172.21.15.92:0/2990099673 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5b7406e970 0x7f5b741c4050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:51.598 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.593+0000 7f5b737fe640 1 --2- 172.21.15.92:0/2990099673 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5b74070280 0x7f5b741c4ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:51.598 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.593+0000 7f5b727fc640 1 --2- 172.21.15.92:0/2990099673 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5b7406f9a0 0x7f5b741c4590 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f5b6800c390 tx=0x7f5b6800c850 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:51.598 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.594+0000 7f5b5b7fe640 1 -- 172.21.15.92:0/2990099673 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5b68017070 con 0x7f5b7406f9a0 2024-09-21T09:41:51.598 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.594+0000 7f5b5b7fe640 1 -- 172.21.15.92:0/2990099673 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f5b680053c0 con 0x7f5b7406f9a0 2024-09-21T09:41:51.598 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.594+0000 7f5b7939c640 1 -- 172.21.15.92:0/2990099673 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5b741263c0 con 0x7f5b7406f9a0 2024-09-21T09:41:51.599 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.594+0000 7f5b5b7fe640 1 -- 172.21.15.92:0/2990099673 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5b68003850 con 0x7f5b7406f9a0 2024-09-21T09:41:51.599 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.594+0000 7f5b7939c640 1 -- 172.21.15.92:0/2990099673 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f5b74123880 con 0x7f5b7406f9a0 2024-09-21T09:41:51.599 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.595+0000 7f5b5b7fe640 1 -- 172.21.15.92:0/2990099673 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 4) ==== 55943+0+0 (secure 0 0 0) 0x7f5b6801a070 con 0x7f5b7406f9a0 2024-09-21T09:41:51.599 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.595+0000 7f5b7939c640 1 -- 172.21.15.92:0/2990099673 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5b7406ed50 con 0x7f5b7406f9a0 2024-09-21T09:41:51.599 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.595+0000 7f5b5b7fe640 1 -- 172.21.15.92:0/2990099673 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(13..13 src has 1..13) ==== 2448+0+0 (secure 0 0 0) 0x7f5b6805caa0 con 0x7f5b7406f9a0 2024-09-21T09:41:51.600 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.599+0000 7f5b5b7fe640 1 -- 172.21.15.92:0/2990099673 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f5b741263c0 con 0x7f5b7406f9a0 2024-09-21T09:41:51.742 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.740+0000 7f5b7939c640 1 -- 172.21.15.92:0/2990099673 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f5b740694d0 con 0x7f5b7406f9a0 2024-09-21T09:41:51.742 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.740+0000 7f5b5b7fe640 1 -- 172.21.15.92:0/2990099673 <== mon.1 v2:172.21.15.92:3301/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v13) ==== 74+0+7006 (secure 0 0 0) 0x7f5b6803bfe0 con 0x7f5b7406f9a0 2024-09-21T09:41:51.743 INFO:teuthology.misc.health.smithi092.stdout: 2024-09-21T09:41:51.743 INFO:teuthology.misc.health.smithi092.stdout:{"epoch":13,"fsid":"5f250e6c-4e36-49b1-ab62-d9454369fa03","created":"2024-09-21T09:41:34.680338+0000","modified":"2024-09-21T09:41:50.808452+0000","last_up_change":"2024-09-21T09:41:48.412163+0000","last_in_change":"2024-09-21T09:41:37.598778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:41:50.040313+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f3314f59-3374-4072-85c9-f8c4e458efc8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6805","nonce":195102377}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6806","nonce":195102377}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6808","nonce":195102377}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6807","nonce":195102377}]},"public_addr":"172.21.15.92:6805/195102377","cluster_addr":"172.21.15.92:6806/195102377","heartbeat_back_addr":"172.21.15.92:6808/195102377","heartbeat_front_addr":"172.21.15.92:6807/195102377","state":["exists","up"]},{"osd":1,"uuid":"cf0486af-ba68-4763-a364-c1f2181604a7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6813","nonce":1359151798}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6814","nonce":1359151798}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6816","nonce":1359151798}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6815","nonce":1359151798}]},"public_addr":"172.21.15.92:6813/1359151798","cluster_addr":"172.21.15.92:6814/1359151798","heartbeat_back_addr":"172.21.15.92:6816/1359151798","heartbeat_front_addr":"172.21.15.92:6815/1359151798","state":["exists","up"]},{"osd":2,"uuid":"5c587421-3409-4469-a2ee-b2ea20cd45e5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6809","nonce":296401696}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6810","nonce":296401696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6812","nonce":296401696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6811","nonce":296401696}]},"public_addr":"172.21.15.92:6809/296401696","cluster_addr":"172.21.15.92:6810/296401696","heartbeat_back_addr":"172.21.15.92:6812/296401696","heartbeat_front_addr":"172.21.15.92:6811/296401696","state":["exists","up"]},{"osd":3,"uuid":"b98663b7-de4d-4c12-ac91-8c33f159da35","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6801","nonce":1829519295}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6802","nonce":1829519295}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6804","nonce":1829519295}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6803","nonce":1829519295}]},"public_addr":"172.21.15.92:6801/1829519295","cluster_addr":"172.21.15.92:6802/1829519295","heartbeat_back_addr":"172.21.15.92:6804/1829519295","heartbeat_front_addr":"172.21.15.92:6803/1829519295","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.934317+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:45.144020+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.895848+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.636219+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:41:51.745 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.742+0000 7f5b7939c640 1 -- 172.21.15.92:0/2990099673 >> v1:172.21.15.92:6800/1280404317 conn(0x7f5b60047f40 legacy=0x7f5b6004a400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:51.745 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.742+0000 7f5b7939c640 1 -- 172.21.15.92:0/2990099673 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5b7406f9a0 msgr2=0x7f5b741c4590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:51.745 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.742+0000 7f5b7939c640 1 --2- 172.21.15.92:0/2990099673 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5b7406f9a0 0x7f5b741c4590 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f5b6800c390 tx=0x7f5b6800c850 comp rx=0 tx=0).stop 2024-09-21T09:41:51.745 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.742+0000 7f5b7939c640 1 -- 172.21.15.92:0/2990099673 shutdown_connections 2024-09-21T09:41:51.745 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.742+0000 7f5b7939c640 1 --2- 172.21.15.92:0/2990099673 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5b74070280 0x7f5b741c4ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:51.745 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.742+0000 7f5b7939c640 1 --2- 172.21.15.92:0/2990099673 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5b7406f9a0 0x7f5b741c4590 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:51.745 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.742+0000 7f5b7939c640 1 --2- 172.21.15.92:0/2990099673 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5b7406e970 0x7f5b741c4050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:51.745 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.742+0000 7f5b7939c640 1 -- 172.21.15.92:0/2990099673 >> 172.21.15.92:0/2990099673 conn(0x7f5b7409c6c0 msgr2=0x7f5b74091400 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:51.745 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.742+0000 7f5b7939c640 1 -- 172.21.15.92:0/2990099673 shutdown_connections 2024-09-21T09:41:51.745 INFO:teuthology.misc.health.smithi092.stderr:2024-09-21T09:41:51.742+0000 7f5b7939c640 1 -- 172.21.15.92:0/2990099673 wait complete. 2024-09-21T09:41:51.754 DEBUG:teuthology.misc:4 of 4 OSDs are up 2024-09-21T09:41:51.754 INFO:tasks.ceph:Creating RBD pool 2024-09-21T09:41:51.754 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph osd pool create rbd 8 2024-09-21T09:41:51.865 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.863+0000 7f599574c640 1 -- 172.21.15.92:0/767278644 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f599012afa0 msgr2=0x7f599012b380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:51.865 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.863+0000 7f599574c640 1 --2- 172.21.15.92:0/767278644 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f599012afa0 0x7f599012b380 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f5980009830 tx=0x7f598002a510 comp rx=0 tx=0).stop 2024-09-21T09:41:51.865 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.863+0000 7f599574c640 1 -- 172.21.15.92:0/767278644 shutdown_connections 2024-09-21T09:41:51.865 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.863+0000 7f599574c640 1 --2- 172.21.15.92:0/767278644 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5990096930 0x7f5990095500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:51.866 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.863+0000 7f599574c640 1 --2- 172.21.15.92:0/767278644 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f599012afa0 0x7f599012b380 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:51.866 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.863+0000 7f599574c640 1 -- 172.21.15.92:0/767278644 >> 172.21.15.92:0/767278644 conn(0x7f599009d620 msgr2=0x7f5990099af0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:51.866 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.863+0000 7f599574c640 1 -- 172.21.15.92:0/767278644 shutdown_connections 2024-09-21T09:41:51.866 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.863+0000 7f599574c640 1 -- 172.21.15.92:0/767278644 wait complete. 2024-09-21T09:41:51.868 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.864+0000 7f599574c640 1 Processor -- start 2024-09-21T09:41:51.868 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.864+0000 7f599574c640 1 -- start start 2024-09-21T09:41:51.868 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f599574c640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5990095a40 0x7f59901220b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:51.868 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f599574c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5990096930 0x7f59901225f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:51.868 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f599574c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f599012afa0 0x7f5990122b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:51.868 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f599574c640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f5990133960 con 0x7f5990096930 2024-09-21T09:41:51.868 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f599574c640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f59901337e0 con 0x7f599012afa0 2024-09-21T09:41:51.868 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f599574c640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f5990133ae0 con 0x7f5990095a40 2024-09-21T09:41:51.868 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f598effd640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5990095a40 0x7f59901220b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:51.868 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f598e7fc640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5990096930 0x7f59901225f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:51.869 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f598effd640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5990095a40 0x7f59901220b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:37866/0 (socket says 172.21.15.92:37866) 2024-09-21T09:41:51.869 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f598effd640 1 -- 172.21.15.92:0/1960974230 learned_addr learned my addr 172.21.15.92:0/1960974230 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:51.869 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f598f7fe640 1 --2- 172.21.15.92:0/1960974230 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f599012afa0 0x7f5990122b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:51.869 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f598e7fc640 1 -- 172.21.15.92:0/1960974230 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5990095a40 msgr2=0x7f59901220b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:51.869 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f598e7fc640 1 --2- 172.21.15.92:0/1960974230 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5990095a40 0x7f59901220b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:51.869 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f598e7fc640 1 -- 172.21.15.92:0/1960974230 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f599012afa0 msgr2=0x7f5990122b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:51.869 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f598e7fc640 1 --2- 172.21.15.92:0/1960974230 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f599012afa0 0x7f5990122b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:51.869 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f598e7fc640 1 -- 172.21.15.92:0/1960974230 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5990125230 con 0x7f5990096930 2024-09-21T09:41:51.869 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f598f7fe640 1 --2- 172.21.15.92:0/1960974230 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f599012afa0 0x7f5990122b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:51.869 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f598effd640 1 --2- 172.21.15.92:0/1960974230 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5990095a40 0x7f59901220b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:51.869 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f598e7fc640 1 --2- 172.21.15.92:0/1960974230 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5990096930 0x7f59901225f0 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f598400e9c0 tx=0x7f598400ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:51.869 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f59777fe640 1 -- 172.21.15.92:0/1960974230 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f598400ccc0 con 0x7f5990096930 2024-09-21T09:41:51.869 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f59777fe640 1 -- 172.21.15.92:0/1960974230 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f5984004020 con 0x7f5990096930 2024-09-21T09:41:51.870 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f599574c640 1 -- 172.21.15.92:0/1960974230 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f59901231c0 con 0x7f5990096930 2024-09-21T09:41:51.870 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.865+0000 7f59777fe640 1 -- 172.21.15.92:0/1960974230 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f598400ce60 con 0x7f5990096930 2024-09-21T09:41:51.870 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.866+0000 7f599574c640 1 -- 172.21.15.92:0/1960974230 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5990072a10 con 0x7f5990096930 2024-09-21T09:41:51.870 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.867+0000 7f599574c640 1 -- 172.21.15.92:0/1960974230 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5990095500 con 0x7f5990096930 2024-09-21T09:41:51.870 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.867+0000 7f59777fe640 1 -- 172.21.15.92:0/1960974230 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 4) ==== 55943+0+0 (secure 0 0 0) 0x7f59840035f0 con 0x7f5990096930 2024-09-21T09:41:51.870 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.867+0000 7f59777fe640 1 -- 172.21.15.92:0/1960974230 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(14..14 src has 1..14) ==== 2459+0+0 (secure 0 0 0) 0x7f598405e020 con 0x7f5990096930 2024-09-21T09:41:51.873 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:51.871+0000 7f59777fe640 1 -- 172.21.15.92:0/1960974230 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f5984004430 con 0x7f5990096930 2024-09-21T09:41:52.037 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.035+0000 7f599574c640 1 -- 172.21.15.92:0/1960974230 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd pool create", "pool": "rbd", "pg_num": 8} v 0) -- 0x7f5990068c30 con 0x7f5990096930 2024-09-21T09:41:52.829 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.827+0000 7f59777fe640 1 -- 172.21.15.92:0/1960974230 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]=0 pool 'rbd' created v15) ==== 109+0+0 (secure 0 0 0) 0x7f598403d4c0 con 0x7f5990096930 2024-09-21T09:41:52.829 INFO:teuthology.orchestra.run.smithi092.stderr:pool 'rbd' created 2024-09-21T09:41:52.831 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.830+0000 7f599574c640 1 -- 172.21.15.92:0/1960974230 >> v1:172.21.15.92:6800/1280404317 conn(0x7f597c047f40 legacy=0x7f597c04a400 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:52.832 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.830+0000 7f599574c640 1 -- 172.21.15.92:0/1960974230 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5990096930 msgr2=0x7f59901225f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:52.832 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.830+0000 7f599574c640 1 --2- 172.21.15.92:0/1960974230 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5990096930 0x7f59901225f0 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f598400e9c0 tx=0x7f598400ee80 comp rx=0 tx=0).stop 2024-09-21T09:41:52.832 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.830+0000 7f599574c640 1 -- 172.21.15.92:0/1960974230 shutdown_connections 2024-09-21T09:41:52.832 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.830+0000 7f599574c640 1 --2- 172.21.15.92:0/1960974230 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f599012afa0 0x7f5990122b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:52.832 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.830+0000 7f599574c640 1 --2- 172.21.15.92:0/1960974230 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5990096930 0x7f59901225f0 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:52.832 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.830+0000 7f599574c640 1 --2- 172.21.15.92:0/1960974230 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5990095a40 0x7f59901220b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:52.833 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.830+0000 7f599574c640 1 -- 172.21.15.92:0/1960974230 >> 172.21.15.92:0/1960974230 conn(0x7f599009d620 msgr2=0x7f5990125440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:52.833 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.831+0000 7f599574c640 1 -- 172.21.15.92:0/1960974230 shutdown_connections 2024-09-21T09:41:52.833 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.831+0000 7f599574c640 1 -- 172.21.15.92:0/1960974230 wait complete. 2024-09-21T09:41:52.847 DEBUG:teuthology.orchestra.run.smithi092:> rbd --cluster ceph pool init rbd 2024-09-21T09:41:52.895 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.893+0000 7f402f312d80 1 -- 172.21.15.92:0/2250753376 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x55fb073600c0 msgr2=0x55fb073624b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:52.895 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.893+0000 7f402f312d80 1 --2- 172.21.15.92:0/2250753376 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x55fb073600c0 0x55fb073624b0 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7f402800a630 tx=0x7f4028045de0 comp rx=0 tx=0).stop 2024-09-21T09:41:52.895 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.893+0000 7f402f312d80 1 -- 172.21.15.92:0/2250753376 shutdown_connections 2024-09-21T09:41:52.895 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.893+0000 7f402f312d80 1 --2- 172.21.15.92:0/2250753376 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x55fb073600c0 0x55fb073624b0 unknown :-1 s=CLOSED pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:52.895 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.893+0000 7f402f312d80 1 --2- 172.21.15.92:0/2250753376 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x55fb07357540 0x55fb0735fb80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:52.895 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.893+0000 7f402f312d80 1 -- 172.21.15.92:0/2250753376 >> 172.21.15.92:0/2250753376 conn(0x55fb072ca060 msgr2=0x55fb072ca470 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:52.896 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.893+0000 7f402f312d80 1 -- 172.21.15.92:0/2250753376 shutdown_connections 2024-09-21T09:41:52.896 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.893+0000 7f402f312d80 1 -- 172.21.15.92:0/2250753376 wait complete. 2024-09-21T09:41:52.897 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.894+0000 7f402f312d80 1 Processor -- start 2024-09-21T09:41:52.897 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.894+0000 7f402f312d80 1 -- start start 2024-09-21T09:41:52.897 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.894+0000 7f402f312d80 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x55fb07357e20 0x55fb0735fa20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:52.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.894+0000 7f402f312d80 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x55fb073600c0 0x55fb07352940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:52.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.894+0000 7f402f312d80 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x55fb07352e80 0x55fb07353300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:52.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.894+0000 7f402f312d80 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x55fb07362e10 con 0x55fb07357e20 2024-09-21T09:41:52.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.894+0000 7f402f312d80 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x55fb07147540 con 0x55fb073600c0 2024-09-21T09:41:52.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.894+0000 7f402f312d80 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x55fb07362f90 con 0x55fb07352e80 2024-09-21T09:41:52.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.894+0000 7f402de08640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x55fb07357e20 0x55fb0735fa20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:52.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.894+0000 7f402f06a640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x55fb07352e80 0x55fb07353300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:52.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.894+0000 7f402d607640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x55fb073600c0 0x55fb07352940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:52.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.894+0000 7f402f06a640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x55fb07352e80 0x55fb07353300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:37900/0 (socket says 172.21.15.92:37900) 2024-09-21T09:41:52.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.894+0000 7f402f06a640 1 -- 172.21.15.92:0/3264086960 learned_addr learned my addr 172.21.15.92:0/3264086960 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:52.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.895+0000 7f402de08640 1 -- 172.21.15.92:0/3264086960 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x55fb07352e80 msgr2=0x55fb07353300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:52.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.895+0000 7f402de08640 1 --2- 172.21.15.92:0/3264086960 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x55fb07352e80 0x55fb07353300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:52.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.895+0000 7f402de08640 1 -- 172.21.15.92:0/3264086960 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x55fb073600c0 msgr2=0x55fb07352940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:52.899 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.895+0000 7f402de08640 1 --2- 172.21.15.92:0/3264086960 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x55fb073600c0 0x55fb07352940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:52.899 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.895+0000 7f402de08640 1 -- 172.21.15.92:0/3264086960 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x55fb0735caa0 con 0x55fb07357e20 2024-09-21T09:41:52.899 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.895+0000 7f402f06a640 1 --2- 172.21.15.92:0/3264086960 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x55fb07352e80 0x55fb07353300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:52.899 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.895+0000 7f402de08640 1 --2- 172.21.15.92:0/3264086960 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x55fb07357e20 0x55fb0735fa20 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f402000be10 tx=0x7f40200095f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:52.899 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.895+0000 7f401dffb640 1 -- 172.21.15.92:0/3264086960 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f402000e620 con 0x55fb07357e20 2024-09-21T09:41:52.899 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.895+0000 7f401dffb640 1 -- 172.21.15.92:0/3264086960 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f4020006050 con 0x55fb07357e20 2024-09-21T09:41:52.899 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.895+0000 7f401dffb640 1 -- 172.21.15.92:0/3264086960 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f402000eb70 con 0x55fb07357e20 2024-09-21T09:41:52.899 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.895+0000 7f402f312d80 1 -- 172.21.15.92:0/3264086960 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x55fb0735c460 con 0x55fb07357e20 2024-09-21T09:41:52.899 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.895+0000 7f402f312d80 1 -- 172.21.15.92:0/3264086960 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x55fb07371ca0 con 0x55fb07357e20 2024-09-21T09:41:52.899 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.896+0000 7f401dffb640 1 -- 172.21.15.92:0/3264086960 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f402000ed10 con 0x55fb07357e20 2024-09-21T09:41:52.900 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.896+0000 7f401dffb640 1 -- 172.21.15.92:0/3264086960 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(15..15 src has 1..15) ==== 2801+0+0 (secure 0 0 0) 0x7f402005c740 con 0x55fb07357e20 2024-09-21T09:41:52.900 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:52.896+0000 7f402f312d80 1 -- 172.21.15.92:0/3264086960 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd pool application enable","pool": "rbd","app": "rbd"} v 0) -- 0x55fb07372080 con 0x55fb07357e20 2024-09-21T09:41:53.830 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:53.828+0000 7f401dffb640 1 -- 172.21.15.92:0/3264086960 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]=0 enabled application 'rbd' on pool 'rbd' v16) ==== 141+0+0 (secure 0 0 0) 0x7f4020012030 con 0x55fb07357e20 2024-09-21T09:41:53.830 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:53.828+0000 7f402f312d80 1 -- 172.21.15.92:0/3264086960 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f4028051070 con 0x55fb07357e20 2024-09-21T09:41:53.837 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:53.835+0000 7f401dffb640 1 -- 172.21.15.92:0/3264086960 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_get_version_reply(handle=1 version=16) ==== 24+0+0 (secure 0 0 0) 0x7f4020039a40 con 0x55fb07357e20 2024-09-21T09:41:53.837 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:53.835+0000 7f401f7fe640 1 -- 172.21.15.92:0/3264086960 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=16}) -- 0x7f4008001e10 con 0x55fb07357e20 2024-09-21T09:41:53.837 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:53.835+0000 7f401dffb640 1 -- 172.21.15.92:0/3264086960 <== mon.0 v2:172.21.15.92:3300/0 8 ==== osd_map(16..16 src has 1..16) ==== 650+0+0 (secure 0 0 0) 0x7f402005b8d0 con 0x55fb07357e20 2024-09-21T09:41:53.837 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:53.835+0000 7f402f312d80 1 -- 172.21.15.92:0/3264086960 --> v1:172.21.15.92:6801/1829519295 -- osd_op(unknown.0.0:1 2.3 2:c4c92e5a:::rbd_trash:head [create] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e16) -- 0x55fb07376f30 con 0x55fb07373420 2024-09-21T09:41:53.850 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:53.848+0000 7f402f06a640 1 -- 172.21.15.92:0/3264086960 <== osd.3 v1:172.21.15.92:6801/1829519295 1 ==== osd_op_reply(1 rbd_trash [create] v16'1 uv1 ondisk = 0) ==== 153+0+0 (unknown 2685446591 0 0) 0x7f402800b040 con 0x55fb07373420 2024-09-21T09:41:53.852 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:53.851+0000 7f402f312d80 1 -- 172.21.15.92:0/3264086960 --> v1:172.21.15.92:6801/1829519295 -- osd_op(unknown.0.0:2 2.2 2:5cea7035:::rbd_info:head [call rbd.metadata_list in=33b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e16) -- 0x55fb073c9220 con 0x55fb07373420 2024-09-21T09:41:53.853 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:53.851+0000 7f402f06a640 1 -- 172.21.15.92:0/3264086960 <== osd.3 v1:172.21.15.92:6801/1829519295 2 ==== osd_op_reply(2 rbd_info [call] v0'0 uv0 ondisk = -2 ((2) No such file or directory)) ==== 152+0+0 (unknown 2026326479 0 0) 0x7f402800b040 con 0x55fb07373420 2024-09-21T09:41:53.853 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:53.851+0000 7f402f312d80 1 -- 172.21.15.92:0/3264086960 --> v1:172.21.15.92:6801/1829519295 -- osd_op(unknown.0.0:3 2.2 2:5cea7035:::rbd_info:head [read 0~0] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e16) -- 0x55fb073c8310 con 0x55fb07373420 2024-09-21T09:41:53.853 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:53.851+0000 7f402f06a640 1 -- 172.21.15.92:0/3264086960 <== osd.3 v1:172.21.15.92:6801/1829519295 3 ==== osd_op_reply(3 rbd_info [read 0~0] v0'0 uv0 ondisk = -2 ((2) No such file or directory)) ==== 152+0+0 (unknown 2039985091 0 0) 0x7f402800b040 con 0x55fb07373420 2024-09-21T09:41:53.853 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:53.851+0000 7f401f7fe640 1 -- 172.21.15.92:0/3264086960 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- pool_op(create unmanaged snap pool 2 tid 4 name v0) -- 0x7f40080034a0 con 0x55fb07357e20 2024-09-21T09:41:54.858 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:54.856+0000 7f401dffb640 1 -- 172.21.15.92:0/3264086960 <== mon.0 v2:172.21.15.92:3300/0 9 ==== pool_op_reply(tid 4 (0) Success v17) ==== 55+0+0 (secure 0 0 0) 0x7f4020002fd0 con 0x55fb07357e20 2024-09-21T09:41:54.859 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:54.856+0000 7f401dffb640 1 -- 172.21.15.92:0/3264086960 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=17}) -- 0x7f401004d450 con 0x55fb07357e20 2024-09-21T09:41:54.859 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:54.857+0000 7f401dffb640 1 -- 172.21.15.92:0/3264086960 <== mon.0 v2:172.21.15.92:3300/0 10 ==== osd_map(17..17 src has 1..17) ==== 626+0+0 (secure 0 0 0) 0x7f4020050070 con 0x55fb07357e20 2024-09-21T09:41:54.859 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:54.857+0000 7f401f7fe640 1 -- 172.21.15.92:0/3264086960 --> v1:172.21.15.92:6801/1829519295 -- osd_op(unknown.0.0:5 2.2 2:5cea7035:::rbd_info:head [create,write 0~8 in=8b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e17) -- 0x7f4008004e50 con 0x55fb07373420 2024-09-21T09:41:54.860 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:54.858+0000 7f402f06a640 1 -- 172.21.15.92:0/3264086960 <== osd.3 v1:172.21.15.92:6801/1829519295 4 ==== osd_op_reply(5 rbd_info [create,write 0~8] v17'1 uv1 ondisk = 0) ==== 194+0+0 (unknown 3115756469 0 0) 0x7f402800b040 con 0x55fb07373420 2024-09-21T09:41:54.860 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:54.858+0000 7f401f7fe640 1 -- 172.21.15.92:0/3264086960 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- pool_op(delete unmanaged snap pool 2 tid 6 name v17) -- 0x7f4008004a90 con 0x55fb07357e20 2024-09-21T09:41:55.860 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.858+0000 7f401dffb640 1 -- 172.21.15.92:0/3264086960 <== mon.0 v2:172.21.15.92:3300/0 11 ==== pool_op_reply(tid 6 (0) Success v18) ==== 43+0+0 (secure 0 0 0) 0x7f402003ba10 con 0x55fb07357e20 2024-09-21T09:41:55.860 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.858+0000 7f401dffb640 1 -- 172.21.15.92:0/3264086960 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=18}) -- 0x7f401004d500 con 0x55fb07357e20 2024-09-21T09:41:55.860 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.859+0000 7f401dffb640 1 -- 172.21.15.92:0/3264086960 <== mon.0 v2:172.21.15.92:3300/0 12 ==== osd_map(18..18 src has 1..18) ==== 654+0+0 (secure 0 0 0) 0x7f4020058020 con 0x55fb07357e20 2024-09-21T09:41:55.861 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.859+0000 7f401f7fe640 1 -- 172.21.15.92:0/3264086960 --> v1:172.21.15.92:6801/1829519295 -- osd_op(unknown.0.0:7 2.2 2:5cea7035:::rbd_info:head [write 0~19 in=19b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e18) -- 0x7f40080056a0 con 0x55fb07373420 2024-09-21T09:41:55.863 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.861+0000 7f402f06a640 1 -- 172.21.15.92:0/3264086960 <== osd.3 v1:172.21.15.92:6801/1829519295 5 ==== osd_op_reply(7 rbd_info [write 0~19] v18'2 uv2 ondisk = 0) ==== 152+0+0 (unknown 4030416863 0 0) 0x7f402800b040 con 0x55fb07373420 2024-09-21T09:41:55.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.862+0000 7f402f312d80 1 -- 172.21.15.92:0/3264086960 >> v1:172.21.15.92:6801/1829519295 conn(0x55fb07373420 legacy=0x55fb07375810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:55.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.862+0000 7f402f312d80 1 -- 172.21.15.92:0/3264086960 >> v1:172.21.15.92:6800/1280404317 conn(0x7f40100468c0 legacy=0x7f4010048d80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:55.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.862+0000 7f402f312d80 1 -- 172.21.15.92:0/3264086960 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x55fb07357e20 msgr2=0x55fb0735fa20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:55.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.862+0000 7f402f312d80 1 --2- 172.21.15.92:0/3264086960 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x55fb07357e20 0x55fb0735fa20 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f402000be10 tx=0x7f40200095f0 comp rx=0 tx=0).stop 2024-09-21T09:41:55.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.862+0000 7f402f06a640 1 -- 172.21.15.92:0/3264086960 reap_dead start 2024-09-21T09:41:55.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.862+0000 7f402f312d80 1 -- 172.21.15.92:0/3264086960 shutdown_connections 2024-09-21T09:41:55.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.862+0000 7f402f312d80 1 -- 172.21.15.92:0/3264086960 >> 172.21.15.92:0/3264086960 conn(0x55fb072ca060 msgr2=0x55fb07355ce0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:55.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.862+0000 7f402f312d80 1 -- 172.21.15.92:0/3264086960 shutdown_connections 2024-09-21T09:41:55.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.862+0000 7f402f312d80 1 -- 172.21.15.92:0/3264086960 wait complete. 2024-09-21T09:41:55.868 INFO:tasks.ceph:Starting mds daemons in cluster ceph... 2024-09-21T09:41:55.868 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config log 1 --format=json 2024-09-21T09:41:55.869 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2024-09-21T09:41:55.975 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.973+0000 7f10b6461640 1 -- 172.21.15.92:0/2477554243 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f10b012b700 msgr2=0x7f10b0127d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:55.975 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.973+0000 7f10b6461640 1 --2- 172.21.15.92:0/2477554243 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f10b012b700 0x7f10b0127d30 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7f10a8005fd0 tx=0x7f10a802a780 comp rx=0 tx=0).stop 2024-09-21T09:41:55.975 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.973+0000 7f10b6461640 1 -- 172.21.15.92:0/2477554243 shutdown_connections 2024-09-21T09:41:55.975 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.973+0000 7f10b6461640 1 --2- 172.21.15.92:0/2477554243 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f10b012b700 0x7f10b0127d30 unknown :-1 s=CLOSED pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:55.975 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.973+0000 7f10b6461640 1 --2- 172.21.15.92:0/2477554243 >> v2:172.21.15.92:3300/0 conn(0x7f10b0092a50 0x7f10b012ad90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:55.975 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.973+0000 7f10b6461640 1 -- 172.21.15.92:0/2477554243 >> 172.21.15.92:0/2477554243 conn(0x7f10b009b370 msgr2=0x7f10b009b780 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:55.976 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.973+0000 7f10b6461640 1 -- 172.21.15.92:0/2477554243 shutdown_connections 2024-09-21T09:41:55.976 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.973+0000 7f10b6461640 1 -- 172.21.15.92:0/2477554243 wait complete. 2024-09-21T09:41:55.977 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.974+0000 7f10b6461640 1 Processor -- start 2024-09-21T09:41:55.978 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.974+0000 7f10b6461640 1 -- start start 2024-09-21T09:41:55.978 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10b6461640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f10b0092a50 0x7f10b00709c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:55.978 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10b6461640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f10b012b2d0 0x7f10b0070f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:55.978 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10b6461640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f10b012b700 0x7f10b0071440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:55.978 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10b6461640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f10b0133760 con 0x7f10b012b700 2024-09-21T09:41:55.978 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10b6461640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f10b01335e0 con 0x7f10b0092a50 2024-09-21T09:41:55.978 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10b6461640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f10b01338e0 con 0x7f10b012b2d0 2024-09-21T09:41:55.978 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10af7fe640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f10b012b2d0 0x7f10b0070f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:55.978 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10affff640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f10b0092a50 0x7f10b00709c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:55.978 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10af7fe640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f10b012b2d0 0x7f10b0070f00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:37912/0 (socket says 172.21.15.92:37912) 2024-09-21T09:41:55.978 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10af7fe640 1 -- 172.21.15.92:0/3278082265 learned_addr learned my addr 172.21.15.92:0/3278082265 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:55.978 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10b49fe640 1 --2- 172.21.15.92:0/3278082265 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f10b012b700 0x7f10b0071440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:55.979 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10af7fe640 1 -- 172.21.15.92:0/3278082265 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f10b0092a50 msgr2=0x7f10b00709c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:55.979 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10af7fe640 1 --2- 172.21.15.92:0/3278082265 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f10b0092a50 0x7f10b00709c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:55.979 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10af7fe640 1 -- 172.21.15.92:0/3278082265 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f10b012b700 msgr2=0x7f10b0071440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:55.979 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10af7fe640 1 --2- 172.21.15.92:0/3278082265 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f10b012b700 0x7f10b0071440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:55.979 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10af7fe640 1 -- 172.21.15.92:0/3278082265 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f10b006be30 con 0x7f10b012b2d0 2024-09-21T09:41:55.979 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10b49fe640 1 --2- 172.21.15.92:0/3278082265 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f10b012b700 0x7f10b0071440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:55.979 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10af7fe640 1 --2- 172.21.15.92:0/3278082265 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f10b012b2d0 0x7f10b0070f00 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7f109800c450 tx=0x7f109800c910 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:55.979 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10acff9640 1 -- 172.21.15.92:0/3278082265 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f10980052d0 con 0x7f10b012b2d0 2024-09-21T09:41:55.979 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10acff9640 1 -- 172.21.15.92:0/3278082265 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f1098005470 con 0x7f10b012b2d0 2024-09-21T09:41:55.979 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10b6461640 1 -- 172.21.15.92:0/3278082265 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f10b0072320 con 0x7f10b012b2d0 2024-09-21T09:41:55.979 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10acff9640 1 -- 172.21.15.92:0/3278082265 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1098012870 con 0x7f10b012b2d0 2024-09-21T09:41:55.979 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.975+0000 7f10b6461640 1 -- 172.21.15.92:0/3278082265 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f10b01c1d60 con 0x7f10b012b2d0 2024-09-21T09:41:55.980 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.976+0000 7f10acff9640 1 -- 172.21.15.92:0/3278082265 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f10980035f0 con 0x7f10b012b2d0 2024-09-21T09:41:55.980 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.977+0000 7f10b6461640 1 -- 172.21.15.92:0/3278082265 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f10b012ad90 con 0x7f10b012b2d0 2024-09-21T09:41:55.980 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.977+0000 7f10acff9640 1 -- 172.21.15.92:0/3278082265 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7f109805d4e0 con 0x7f10b012b2d0 2024-09-21T09:41:55.982 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:55.981+0000 7f10acff9640 1 -- 172.21.15.92:0/3278082265 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f109805c630 con 0x7f10b012b2d0 2024-09-21T09:41:56.124 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.122+0000 7f10b6461640 1 -- 172.21.15.92:0/3278082265 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "config log", "num": 1, "format": "json"} v 0) -- 0x7f10b0073ee0 con 0x7f10b012b2d0 2024-09-21T09:41:56.125 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.123+0000 7f10acff9640 1 -- 172.21.15.92:0/3278082265 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mon_command_ack([{"prefix": "config log", "num": 1, "format": "json"}]=0 v23) ==== 86+0+160 (secure 0 0 0) 0x7f109803c770 con 0x7f10b012b2d0 2024-09-21T09:41:56.125 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:56.126 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.124+0000 7f10b6461640 1 -- 172.21.15.92:0/3278082265 >> v1:172.21.15.92:6800/1280404317 conn(0x7f1090047fd0 legacy=0x7f109004a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:56.126 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.124+0000 7f10b6461640 1 -- 172.21.15.92:0/3278082265 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f10b012b2d0 msgr2=0x7f10b0070f00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:56.126 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.124+0000 7f10b6461640 1 --2- 172.21.15.92:0/3278082265 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f10b012b2d0 0x7f10b0070f00 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7f109800c450 tx=0x7f109800c910 comp rx=0 tx=0).stop 2024-09-21T09:41:56.127 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.125+0000 7f10b6461640 1 -- 172.21.15.92:0/3278082265 shutdown_connections 2024-09-21T09:41:56.127 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.125+0000 7f10b6461640 1 --2- 172.21.15.92:0/3278082265 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f10b012b700 0x7f10b0071440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.127 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.125+0000 7f10b6461640 1 --2- 172.21.15.92:0/3278082265 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f10b012b2d0 0x7f10b0070f00 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.127 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.125+0000 7f10b6461640 1 --2- 172.21.15.92:0/3278082265 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f10b0092a50 0x7f10b00709c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.127 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.125+0000 7f10b6461640 1 -- 172.21.15.92:0/3278082265 >> 172.21.15.92:0/3278082265 conn(0x7f10b009b370 msgr2=0x7f10b0129a10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:56.127 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.125+0000 7f10b6461640 1 -- 172.21.15.92:0/3278082265 shutdown_connections 2024-09-21T09:41:56.127 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.125+0000 7f10b6461640 1 -- 172.21.15.92:0/3278082265 wait complete. 2024-09-21T09:41:56.137 INFO:teuthology.orchestra.run.smithi092.stdout:[{"version":23,"timestamp":"2024-09-21T09:41:47.456887+0000","name":"","changes":[{"name":"osd.2/osd_mclock_max_capacity_iops_ssd","new_value":"5932.819082"}]}] 2024-09-21T09:41:56.137 INFO:tasks.ceph_manager:config epoch is 23 2024-09-21T09:41:56.137 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T09:41:56.138 INFO:tasks.ceph.ceph_manager.ceph:waiting for mgr available 2024-09-21T09:41:56.138 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mgr dump --format=json 2024-09-21T09:41:56.237 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.234+0000 7f1a34296640 1 -- 172.21.15.92:0/2178285699 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f1a2c12c1d0 msgr2=0x7f1a2c12c620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:56.237 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.234+0000 7f1a34296640 1 --2- 172.21.15.92:0/2178285699 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f1a2c12c1d0 0x7f1a2c12c620 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f1a24009be0 tx=0x7f1a2402a760 comp rx=0 tx=0).stop 2024-09-21T09:41:56.237 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.234+0000 7f1a34296640 1 -- 172.21.15.92:0/2178285699 shutdown_connections 2024-09-21T09:41:56.237 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.234+0000 7f1a34296640 1 --2- 172.21.15.92:0/2178285699 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f1a2c12c1d0 0x7f1a2c12c620 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.237 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.234+0000 7f1a34296640 1 --2- 172.21.15.92:0/2178285699 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1a2c095f50 0x7f1a2c096330 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.237 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.234+0000 7f1a34296640 1 -- 172.21.15.92:0/2178285699 >> 172.21.15.92:0/2178285699 conn(0x7f1a2c09c590 msgr2=0x7f1a2c09c9a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:56.237 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.234+0000 7f1a34296640 1 -- 172.21.15.92:0/2178285699 shutdown_connections 2024-09-21T09:41:56.237 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.234+0000 7f1a34296640 1 -- 172.21.15.92:0/2178285699 wait complete. 2024-09-21T09:41:56.237 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.235+0000 7f1a34296640 1 Processor -- start 2024-09-21T09:41:56.238 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.235+0000 7f1a34296640 1 -- start start 2024-09-21T09:41:56.239 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a34296640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1a2c095f50 0x7f1a2c1c1dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:56.239 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a34296640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f1a2c096870 0x7f1a2c1c2300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:56.240 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a34296640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1a2c12c1d0 0x7f1a2c06f9d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:56.240 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a34296640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f1a2c1347e0 con 0x7f1a2c095f50 2024-09-21T09:41:56.240 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a34296640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f1a2c134660 con 0x7f1a2c096870 2024-09-21T09:41:56.240 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a34296640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f1a2c134960 con 0x7f1a2c12c1d0 2024-09-21T09:41:56.240 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a32032640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1a2c095f50 0x7f1a2c1c1dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:56.240 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a32833640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1a2c12c1d0 0x7f1a2c06f9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:56.240 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a32032640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1a2c095f50 0x7f1a2c1c1dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:40252/0 (socket says 172.21.15.92:40252) 2024-09-21T09:41:56.240 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a32032640 1 -- 172.21.15.92:0/3966947035 learned_addr learned my addr 172.21.15.92:0/3966947035 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:56.240 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a31831640 1 --2- 172.21.15.92:0/3966947035 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f1a2c096870 0x7f1a2c1c2300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:56.240 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a32032640 1 -- 172.21.15.92:0/3966947035 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1a2c12c1d0 msgr2=0x7f1a2c06f9d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:56.240 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a32032640 1 --2- 172.21.15.92:0/3966947035 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1a2c12c1d0 0x7f1a2c06f9d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.240 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a32032640 1 -- 172.21.15.92:0/3966947035 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f1a2c096870 msgr2=0x7f1a2c1c2300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:56.240 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a32032640 1 --2- 172.21.15.92:0/3966947035 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f1a2c096870 0x7f1a2c1c2300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.241 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a32032640 1 -- 172.21.15.92:0/3966947035 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1a2c0726f0 con 0x7f1a2c095f50 2024-09-21T09:41:56.241 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a32833640 1 --2- 172.21.15.92:0/3966947035 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1a2c12c1d0 0x7f1a2c06f9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:56.241 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a32032640 1 --2- 172.21.15.92:0/3966947035 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1a2c095f50 0x7f1a2c1c1dc0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f1a2800ea60 tx=0x7f1a2800ef20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:56.241 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a22ffd640 1 -- 172.21.15.92:0/3966947035 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a2800cde0 con 0x7f1a2c095f50 2024-09-21T09:41:56.241 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a22ffd640 1 -- 172.21.15.92:0/3966947035 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f1a28004020 con 0x7f1a2c095f50 2024-09-21T09:41:56.241 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a22ffd640 1 -- 172.21.15.92:0/3966947035 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1a28004470 con 0x7f1a2c095f50 2024-09-21T09:41:56.241 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a34296640 1 -- 172.21.15.92:0/3966947035 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1a2c0713f0 con 0x7f1a2c095f50 2024-09-21T09:41:56.241 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.236+0000 7f1a34296640 1 -- 172.21.15.92:0/3966947035 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1a2c073370 con 0x7f1a2c095f50 2024-09-21T09:41:56.241 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.237+0000 7f1a34296640 1 -- 172.21.15.92:0/3966947035 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1a2c126780 con 0x7f1a2c095f50 2024-09-21T09:41:56.241 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.238+0000 7f1a22ffd640 1 -- 172.21.15.92:0/3966947035 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f1a280035f0 con 0x7f1a2c095f50 2024-09-21T09:41:56.241 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.238+0000 7f1a22ffd640 1 -- 172.21.15.92:0/3966947035 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7f1a2805e1d0 con 0x7f1a2c095f50 2024-09-21T09:41:56.243 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.242+0000 7f1a22ffd640 1 -- 172.21.15.92:0/3966947035 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f1a2c126780 con 0x7f1a2c095f50 2024-09-21T09:41:56.410 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.408+0000 7f1a34296640 1 -- 172.21.15.92:0/3966947035 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "mgr dump", "format": "json"} v 0) -- 0x7f1a2c073ee0 con 0x7f1a2c095f50 2024-09-21T09:41:56.412 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.410+0000 7f1a22ffd640 1 -- 172.21.15.92:0/3966947035 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json"}]=0 v5) ==== 74+0+108899 (secure 0 0 0) 0x7f1a2805d320 con 0x7f1a2c095f50 2024-09-21T09:41:56.413 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:56.415 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.412+0000 7f1a34296640 1 -- 172.21.15.92:0/3966947035 >> v1:172.21.15.92:6800/1280404317 conn(0x7f1a10048000 legacy=0x7f1a1004a4c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:56.415 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.412+0000 7f1a34296640 1 -- 172.21.15.92:0/3966947035 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1a2c095f50 msgr2=0x7f1a2c1c1dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:56.415 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.412+0000 7f1a34296640 1 --2- 172.21.15.92:0/3966947035 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1a2c095f50 0x7f1a2c1c1dc0 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f1a2800ea60 tx=0x7f1a2800ef20 comp rx=0 tx=0).stop 2024-09-21T09:41:56.415 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.412+0000 7f1a34296640 1 -- 172.21.15.92:0/3966947035 shutdown_connections 2024-09-21T09:41:56.415 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.412+0000 7f1a34296640 1 --2- 172.21.15.92:0/3966947035 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1a2c12c1d0 0x7f1a2c06f9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.415 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.412+0000 7f1a34296640 1 --2- 172.21.15.92:0/3966947035 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f1a2c096870 0x7f1a2c1c2300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.415 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.412+0000 7f1a34296640 1 --2- 172.21.15.92:0/3966947035 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1a2c095f50 0x7f1a2c1c1dc0 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.415 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.412+0000 7f1a34296640 1 -- 172.21.15.92:0/3966947035 >> 172.21.15.92:0/3966947035 conn(0x7f1a2c09c590 msgr2=0x7f1a2c129000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:56.416 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.413+0000 7f1a34296640 1 -- 172.21.15.92:0/3966947035 shutdown_connections 2024-09-21T09:41:56.416 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.413+0000 7f1a34296640 1 -- 172.21.15.92:0/3966947035 wait complete. 2024-09-21T09:41:56.424 INFO:teuthology.orchestra.run.smithi092.stdout:{"epoch":5,"flags":0,"active_gid":4146,"active_name":"y","active_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6800","nonce":1280404317}]},"active_addr":"172.21.15.92:6800/1280404317","active_change":"2024-09-21T09:41:38.005482+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":4120,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.92:0","nonce":3587832392}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.92:0","nonce":230938560}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.92:0","nonce":2147030555}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.92:0","nonce":162460762}]}]} 2024-09-21T09:41:56.427 INFO:tasks.ceph.ceph_manager.ceph:mgr available! 2024-09-21T09:41:56.427 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up 2024-09-21T09:41:56.427 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:41:56.528 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.526+0000 7f12d1f9c640 1 -- 172.21.15.92:0/2096430410 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f12cc131700 msgr2=0x7f12cc133af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:56.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.526+0000 7f12d1f9c640 1 --2- 172.21.15.92:0/2096430410 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f12cc131700 0x7f12cc133af0 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7f12c4005fd0 tx=0x7f12c402a980 comp rx=0 tx=0).stop 2024-09-21T09:41:56.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.526+0000 7f12d1f9c640 1 -- 172.21.15.92:0/2096430410 shutdown_connections 2024-09-21T09:41:56.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.526+0000 7f12d1f9c640 1 --2- 172.21.15.92:0/2096430410 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f12cc131700 0x7f12cc133af0 unknown :-1 s=CLOSED pgs=34 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.526+0000 7f12d1f9c640 1 --2- 172.21.15.92:0/2096430410 >> v2:172.21.15.92:3300/0 conn(0x7f12cc12edd0 0x7f12cc1311c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.526+0000 7f12d1f9c640 1 -- 172.21.15.92:0/2096430410 >> 172.21.15.92:0/2096430410 conn(0x7f12cc09c590 msgr2=0x7f12cc09c9a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:56.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.527+0000 7f12d1f9c640 1 -- 172.21.15.92:0/2096430410 shutdown_connections 2024-09-21T09:41:56.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.527+0000 7f12d1f9c640 1 -- 172.21.15.92:0/2096430410 wait complete. 2024-09-21T09:41:56.531 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.527+0000 7f12d1f9c640 1 Processor -- start 2024-09-21T09:41:56.531 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.528+0000 7f12d1f9c640 1 -- start start 2024-09-21T09:41:56.531 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.528+0000 7f12d1f9c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f12cc095f50 0x7f12cc06d570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:56.531 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.528+0000 7f12d1f9c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f12cc12edd0 0x7f12cc06dab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:56.531 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.528+0000 7f12d1f9c640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f12cc131700 0x7f12cc06dff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:56.531 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.528+0000 7f12d1f9c640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f12cc134740 con 0x7f12cc12edd0 2024-09-21T09:41:56.531 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.528+0000 7f12d1f9c640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f12cc1345c0 con 0x7f12cc095f50 2024-09-21T09:41:56.531 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.528+0000 7f12d1f9c640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f12cc1348c0 con 0x7f12cc131700 2024-09-21T09:41:56.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.528+0000 7f12caffd640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f12cc12edd0 0x7f12cc06dab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:56.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.528+0000 7f12cbfff640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f12cc131700 0x7f12cc06dff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:56.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.528+0000 7f12cb7fe640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f12cc095f50 0x7f12cc06d570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:56.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.528+0000 7f12caffd640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f12cc12edd0 0x7f12cc06dab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:40272/0 (socket says 172.21.15.92:40272) 2024-09-21T09:41:56.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.528+0000 7f12caffd640 1 -- 172.21.15.92:0/2085319801 learned_addr learned my addr 172.21.15.92:0/2085319801 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:56.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.528+0000 7f12caffd640 1 -- 172.21.15.92:0/2085319801 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f12cc131700 msgr2=0x7f12cc06dff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:56.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.528+0000 7f12caffd640 1 --2- 172.21.15.92:0/2085319801 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f12cc131700 0x7f12cc06dff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.528+0000 7f12caffd640 1 -- 172.21.15.92:0/2085319801 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f12cc095f50 msgr2=0x7f12cc06d570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:56.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.528+0000 7f12caffd640 1 --2- 172.21.15.92:0/2085319801 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f12cc095f50 0x7f12cc06d570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.528+0000 7f12caffd640 1 -- 172.21.15.92:0/2085319801 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f12cc125230 con 0x7f12cc12edd0 2024-09-21T09:41:56.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.528+0000 7f12cb7fe640 1 --2- 172.21.15.92:0/2085319801 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f12cc095f50 0x7f12cc06d570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:56.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.528+0000 7f12caffd640 1 --2- 172.21.15.92:0/2085319801 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f12cc12edd0 0x7f12cc06dab0 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f12c000ea60 tx=0x7f12c000ef20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:56.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.529+0000 7f12b3fff640 1 -- 172.21.15.92:0/2085319801 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f12c000cde0 con 0x7f12cc12edd0 2024-09-21T09:41:56.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.529+0000 7f12b3fff640 1 -- 172.21.15.92:0/2085319801 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f12c0004020 con 0x7f12cc12edd0 2024-09-21T09:41:56.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.529+0000 7f12d1f9c640 1 -- 172.21.15.92:0/2085319801 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f12cc125b40 con 0x7f12cc12edd0 2024-09-21T09:41:56.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.529+0000 7f12b3fff640 1 -- 172.21.15.92:0/2085319801 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f12c0004450 con 0x7f12cc12edd0 2024-09-21T09:41:56.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.529+0000 7f12d1f9c640 1 -- 172.21.15.92:0/2085319801 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f12cc06ed60 con 0x7f12cc12edd0 2024-09-21T09:41:56.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.530+0000 7f12d1f9c640 1 -- 172.21.15.92:0/2085319801 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f12cc06eff0 con 0x7f12cc12edd0 2024-09-21T09:41:56.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.530+0000 7f12b3fff640 1 -- 172.21.15.92:0/2085319801 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f12c00035f0 con 0x7f12cc12edd0 2024-09-21T09:41:56.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.530+0000 7f12b3fff640 1 -- 172.21.15.92:0/2085319801 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7f12c005e1f0 con 0x7f12cc12edd0 2024-09-21T09:41:56.535 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.534+0000 7f12b3fff640 1 -- 172.21.15.92:0/2085319801 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f12c005d340 con 0x7f12cc12edd0 2024-09-21T09:41:56.675 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.673+0000 7f12d1f9c640 1 -- 172.21.15.92:0/2085319801 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f12cc096330 con 0x7f12cc12edd0 2024-09-21T09:41:56.675 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.674+0000 7f12b3fff640 1 -- 172.21.15.92:0/2085319801 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v18) ==== 74+0+8901 (secure 0 0 0) 0x7f12c003d420 con 0x7f12cc12edd0 2024-09-21T09:41:56.675 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:56.676 INFO:teuthology.orchestra.run.smithi092.stdout:{"epoch":18,"fsid":"5f250e6c-4e36-49b1-ab62-d9454369fa03","created":"2024-09-21T09:41:34.680338+0000","modified":"2024-09-21T09:41:55.850255+0000","last_up_change":"2024-09-21T09:41:48.412163+0000","last_in_change":"2024-09-21T09:41:37.598778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:41:50.040313+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:41:52.054798+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f3314f59-3374-4072-85c9-f8c4e458efc8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6805","nonce":195102377}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6806","nonce":195102377}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6808","nonce":195102377}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6807","nonce":195102377}]},"public_addr":"172.21.15.92:6805/195102377","cluster_addr":"172.21.15.92:6806/195102377","heartbeat_back_addr":"172.21.15.92:6808/195102377","heartbeat_front_addr":"172.21.15.92:6807/195102377","state":["exists","up"]},{"osd":1,"uuid":"cf0486af-ba68-4763-a364-c1f2181604a7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6813","nonce":1359151798}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6814","nonce":1359151798}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6816","nonce":1359151798}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6815","nonce":1359151798}]},"public_addr":"172.21.15.92:6813/1359151798","cluster_addr":"172.21.15.92:6814/1359151798","heartbeat_back_addr":"172.21.15.92:6816/1359151798","heartbeat_front_addr":"172.21.15.92:6815/1359151798","state":["exists","up"]},{"osd":2,"uuid":"5c587421-3409-4469-a2ee-b2ea20cd45e5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6809","nonce":296401696}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6810","nonce":296401696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6812","nonce":296401696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6811","nonce":296401696}]},"public_addr":"172.21.15.92:6809/296401696","cluster_addr":"172.21.15.92:6810/296401696","heartbeat_back_addr":"172.21.15.92:6812/296401696","heartbeat_front_addr":"172.21.15.92:6811/296401696","state":["exists","up"]},{"osd":3,"uuid":"b98663b7-de4d-4c12-ac91-8c33f159da35","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6801","nonce":1829519295}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6802","nonce":1829519295}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6804","nonce":1829519295}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6803","nonce":1829519295}]},"public_addr":"172.21.15.92:6801/1829519295","cluster_addr":"172.21.15.92:6802/1829519295","heartbeat_back_addr":"172.21.15.92:6804/1829519295","heartbeat_front_addr":"172.21.15.92:6803/1829519295","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.934317+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:45.144020+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.895848+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.636219+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:41:56.677 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.676+0000 7f12d1f9c640 1 -- 172.21.15.92:0/2085319801 >> v1:172.21.15.92:6800/1280404317 conn(0x7f12b8048000 legacy=0x7f12b804a4c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:56.678 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.676+0000 7f12d1f9c640 1 -- 172.21.15.92:0/2085319801 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f12cc12edd0 msgr2=0x7f12cc06dab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:56.678 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.676+0000 7f12d1f9c640 1 --2- 172.21.15.92:0/2085319801 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f12cc12edd0 0x7f12cc06dab0 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f12c000ea60 tx=0x7f12c000ef20 comp rx=0 tx=0).stop 2024-09-21T09:41:56.678 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.676+0000 7f12d1f9c640 1 -- 172.21.15.92:0/2085319801 shutdown_connections 2024-09-21T09:41:56.678 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.676+0000 7f12d1f9c640 1 --2- 172.21.15.92:0/2085319801 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f12cc131700 0x7f12cc06dff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.678 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.676+0000 7f12d1f9c640 1 --2- 172.21.15.92:0/2085319801 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f12cc12edd0 0x7f12cc06dab0 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.678 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.676+0000 7f12d1f9c640 1 --2- 172.21.15.92:0/2085319801 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f12cc095f50 0x7f12cc06d570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.678 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.676+0000 7f12d1f9c640 1 -- 172.21.15.92:0/2085319801 >> 172.21.15.92:0/2085319801 conn(0x7f12cc09c590 msgr2=0x7f12cc12f1b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:56.678 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.676+0000 7f12d1f9c640 1 -- 172.21.15.92:0/2085319801 shutdown_connections 2024-09-21T09:41:56.678 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.676+0000 7f12d1f9c640 1 -- 172.21.15.92:0/2085319801 wait complete. 2024-09-21T09:41:56.688 INFO:tasks.ceph.ceph_manager.ceph:all up! 2024-09-21T09:41:56.688 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:41:56.789 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.784+0000 7fbca8ed2640 1 -- 172.21.15.92:0/1435418786 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbca406fd80 msgr2=0x7fbca40960b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:56.789 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.784+0000 7fbca8ed2640 1 --2- 172.21.15.92:0/1435418786 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbca406fd80 0x7fbca40960b0 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7fbc98009860 tx=0x7fbc9802a3d0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.789 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.784+0000 7fbca8ed2640 1 -- 172.21.15.92:0/1435418786 shutdown_connections 2024-09-21T09:41:56.789 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.784+0000 7fbca8ed2640 1 --2- 172.21.15.92:0/1435418786 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbca406fd80 0x7fbca40960b0 unknown :-1 s=CLOSED pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.789 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.784+0000 7fbca8ed2640 1 --2- 172.21.15.92:0/1435418786 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbca406f460 0x7fbca406f840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.790 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.784+0000 7fbca8ed2640 1 -- 172.21.15.92:0/1435418786 >> 172.21.15.92:0/1435418786 conn(0x7fbca409a9b0 msgr2=0x7fbca409ad90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:56.790 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.784+0000 7fbca8ed2640 1 -- 172.21.15.92:0/1435418786 shutdown_connections 2024-09-21T09:41:56.790 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.784+0000 7fbca8ed2640 1 -- 172.21.15.92:0/1435418786 wait complete. 2024-09-21T09:41:56.790 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.785+0000 7fbca8ed2640 1 Processor -- start 2024-09-21T09:41:56.790 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.785+0000 7fbca8ed2640 1 -- start start 2024-09-21T09:41:56.790 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.786+0000 7fbca8ed2640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbca406f460 0x7fbca4126930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:56.790 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.786+0000 7fbca8ed2640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbca406fd80 0x7fbca4126e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:56.790 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.786+0000 7fbca8ed2640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbca40965f0 0x7fbca41273b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:56.790 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.786+0000 7fbca8ed2640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fbca4133950 con 0x7fbca406f460 2024-09-21T09:41:56.791 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.786+0000 7fbca8ed2640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fbca41337d0 con 0x7fbca406fd80 2024-09-21T09:41:56.791 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.786+0000 7fbca8ed2640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fbca4133ad0 con 0x7fbca40965f0 2024-09-21T09:41:56.791 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.786+0000 7fbca1d9b640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbca406fd80 0x7fbca4126e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:56.791 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.786+0000 7fbca259c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbca406f460 0x7fbca4126930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:56.791 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.786+0000 7fbca1d9b640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbca406fd80 0x7fbca4126e70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:40900/0 (socket says 172.21.15.92:40900) 2024-09-21T09:41:56.791 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.786+0000 7fbca259c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbca406f460 0x7fbca4126930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:40288/0 (socket says 172.21.15.92:40288) 2024-09-21T09:41:56.791 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.786+0000 7fbca2d9d640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbca40965f0 0x7fbca41273b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:56.791 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.786+0000 7fbca1d9b640 1 -- 172.21.15.92:0/4227697179 learned_addr learned my addr 172.21.15.92:0/4227697179 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:56.791 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.786+0000 7fbca1d9b640 1 -- 172.21.15.92:0/4227697179 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbca40965f0 msgr2=0x7fbca41273b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:56.791 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.786+0000 7fbca1d9b640 1 --2- 172.21.15.92:0/4227697179 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbca40965f0 0x7fbca41273b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.792 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.786+0000 7fbca1d9b640 1 -- 172.21.15.92:0/4227697179 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbca406f460 msgr2=0x7fbca4126930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:56.792 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.786+0000 7fbca1d9b640 1 --2- 172.21.15.92:0/4227697179 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbca406f460 0x7fbca4126930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.792 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.786+0000 7fbca1d9b640 1 -- 172.21.15.92:0/4227697179 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbca4127cd0 con 0x7fbca406fd80 2024-09-21T09:41:56.792 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.786+0000 7fbca1d9b640 1 --2- 172.21.15.92:0/4227697179 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbca406fd80 0x7fbca4126e70 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7fbc98009980 tx=0x7fbc98032c10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:56.792 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.786+0000 7fbc92ffd640 1 -- 172.21.15.92:0/4227697179 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbc98040070 con 0x7fbca406fd80 2024-09-21T09:41:56.792 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.786+0000 7fbc92ffd640 1 -- 172.21.15.92:0/4227697179 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fbc98005310 con 0x7fbca406fd80 2024-09-21T09:41:56.792 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.786+0000 7fbca8ed2640 1 -- 172.21.15.92:0/4227697179 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbca406e5f0 con 0x7fbca406fd80 2024-09-21T09:41:56.792 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.786+0000 7fbc92ffd640 1 -- 172.21.15.92:0/4227697179 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbc98005c90 con 0x7fbca406fd80 2024-09-21T09:41:56.792 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.787+0000 7fbca8ed2640 1 -- 172.21.15.92:0/4227697179 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fbca41287b0 con 0x7fbca406fd80 2024-09-21T09:41:56.792 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.787+0000 7fbc92ffd640 1 -- 172.21.15.92:0/4227697179 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fbc98039040 con 0x7fbca406fd80 2024-09-21T09:41:56.792 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.787+0000 7fbca8ed2640 1 -- 172.21.15.92:0/4227697179 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbca406f840 con 0x7fbca406fd80 2024-09-21T09:41:56.793 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.788+0000 7fbc92ffd640 1 -- 172.21.15.92:0/4227697179 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7fbc9807eac0 con 0x7fbca406fd80 2024-09-21T09:41:56.793 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.792+0000 7fbc92ffd640 1 -- 172.21.15.92:0/4227697179 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7fbc9807dc10 con 0x7fbca406fd80 2024-09-21T09:41:56.936 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.934+0000 7fbca8ed2640 1 -- 172.21.15.92:0/4227697179 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fbca4127ff0 con 0x7fbca406fd80 2024-09-21T09:41:56.937 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.935+0000 7fbc92ffd640 1 -- 172.21.15.92:0/4227697179 <== mon.1 v2:172.21.15.92:3301/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v18) ==== 74+0+8901 (secure 0 0 0) 0x7fbc9807d750 con 0x7fbca406fd80 2024-09-21T09:41:56.937 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:41:56.937 INFO:teuthology.orchestra.run.smithi092.stdout:{"epoch":18,"fsid":"5f250e6c-4e36-49b1-ab62-d9454369fa03","created":"2024-09-21T09:41:34.680338+0000","modified":"2024-09-21T09:41:55.850255+0000","last_up_change":"2024-09-21T09:41:48.412163+0000","last_in_change":"2024-09-21T09:41:37.598778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:41:50.040313+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:41:52.054798+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f3314f59-3374-4072-85c9-f8c4e458efc8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6805","nonce":195102377}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6806","nonce":195102377}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6808","nonce":195102377}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6807","nonce":195102377}]},"public_addr":"172.21.15.92:6805/195102377","cluster_addr":"172.21.15.92:6806/195102377","heartbeat_back_addr":"172.21.15.92:6808/195102377","heartbeat_front_addr":"172.21.15.92:6807/195102377","state":["exists","up"]},{"osd":1,"uuid":"cf0486af-ba68-4763-a364-c1f2181604a7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6813","nonce":1359151798}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6814","nonce":1359151798}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6816","nonce":1359151798}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6815","nonce":1359151798}]},"public_addr":"172.21.15.92:6813/1359151798","cluster_addr":"172.21.15.92:6814/1359151798","heartbeat_back_addr":"172.21.15.92:6816/1359151798","heartbeat_front_addr":"172.21.15.92:6815/1359151798","state":["exists","up"]},{"osd":2,"uuid":"5c587421-3409-4469-a2ee-b2ea20cd45e5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6809","nonce":296401696}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6810","nonce":296401696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6812","nonce":296401696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6811","nonce":296401696}]},"public_addr":"172.21.15.92:6809/296401696","cluster_addr":"172.21.15.92:6810/296401696","heartbeat_back_addr":"172.21.15.92:6812/296401696","heartbeat_front_addr":"172.21.15.92:6811/296401696","state":["exists","up"]},{"osd":3,"uuid":"b98663b7-de4d-4c12-ac91-8c33f159da35","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6801","nonce":1829519295}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6802","nonce":1829519295}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6804","nonce":1829519295}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6803","nonce":1829519295}]},"public_addr":"172.21.15.92:6801/1829519295","cluster_addr":"172.21.15.92:6802/1829519295","heartbeat_back_addr":"172.21.15.92:6804/1829519295","heartbeat_front_addr":"172.21.15.92:6803/1829519295","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.934317+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:45.144020+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.895848+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.636219+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:41:56.938 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.936+0000 7fbca8ed2640 1 -- 172.21.15.92:0/4227697179 >> v1:172.21.15.92:6800/1280404317 conn(0x7fbc84047fd0 legacy=0x7fbc8404a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:56.938 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.936+0000 7fbca8ed2640 1 -- 172.21.15.92:0/4227697179 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbca406fd80 msgr2=0x7fbca4126e70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:56.939 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.936+0000 7fbca8ed2640 1 --2- 172.21.15.92:0/4227697179 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbca406fd80 0x7fbca4126e70 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7fbc98009980 tx=0x7fbc98032c10 comp rx=0 tx=0).stop 2024-09-21T09:41:56.939 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.936+0000 7fbca8ed2640 1 -- 172.21.15.92:0/4227697179 shutdown_connections 2024-09-21T09:41:56.939 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.936+0000 7fbca8ed2640 1 --2- 172.21.15.92:0/4227697179 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbca40965f0 0x7fbca41273b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.939 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.936+0000 7fbca8ed2640 1 --2- 172.21.15.92:0/4227697179 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbca406fd80 0x7fbca4126e70 unknown :-1 s=CLOSED pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.939 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.936+0000 7fbca8ed2640 1 --2- 172.21.15.92:0/4227697179 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbca406f460 0x7fbca4126930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:56.939 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.936+0000 7fbca8ed2640 1 -- 172.21.15.92:0/4227697179 >> 172.21.15.92:0/4227697179 conn(0x7fbca409a9b0 msgr2=0x7fbca412edc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:56.939 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.937+0000 7fbca8ed2640 1 -- 172.21.15.92:0/4227697179 shutdown_connections 2024-09-21T09:41:56.939 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:56.937+0000 7fbca8ed2640 1 -- 172.21.15.92:0/4227697179 wait complete. 2024-09-21T09:41:56.949 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 flush_pg_stats 2024-09-21T09:41:56.950 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 flush_pg_stats 2024-09-21T09:41:56.950 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 flush_pg_stats 2024-09-21T09:41:56.950 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 flush_pg_stats 2024-09-21T09:41:57.057 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7fe6306e7640 1 -- 172.21.15.92:0/984639091 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe628095f10 msgr2=0x7fe6280962f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.057 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7fe6306e7640 1 --2- 172.21.15.92:0/984639091 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe628095f10 0x7fe6280962f0 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7fe624005fd0 tx=0x7fe62402a4b0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.057 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7f313fd2c640 1 -- 172.21.15.92:0/33888621 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f313808ce10 msgr2=0x7f31381330b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.057 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7f313fd2c640 1 --2- 172.21.15.92:0/33888621 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f313808ce10 0x7f31381330b0 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7f3128009900 tx=0x7f312802a490 comp rx=0 tx=0).stop 2024-09-21T09:41:57.058 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7fe6306e7640 1 -- 172.21.15.92:0/984639091 shutdown_connections 2024-09-21T09:41:57.058 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7fe6306e7640 1 --2- 172.21.15.92:0/984639091 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe628096830 0x7fe62812ba20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.058 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7fe6306e7640 1 --2- 172.21.15.92:0/984639091 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe628095f10 0x7fe6280962f0 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.058 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7fe6306e7640 1 -- 172.21.15.92:0/984639091 >> 172.21.15.92:0/984639091 conn(0x7fe62809c590 msgr2=0x7fe62809c9a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:57.058 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7f655f53e640 1 -- 172.21.15.92:0/2323923172 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6558093cf0 msgr2=0x7f6558094cd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.059 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7f313fd2c640 1 -- 172.21.15.92:0/33888621 shutdown_connections 2024-09-21T09:41:57.059 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7f313fd2c640 1 --2- 172.21.15.92:0/33888621 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f31381335f0 0x7f31381359e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.059 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7f313fd2c640 1 --2- 172.21.15.92:0/33888621 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f313808ce10 0x7f31381330b0 unknown :-1 s=CLOSED pgs=34 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.059 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7f313fd2c640 1 -- 172.21.15.92:0/33888621 >> 172.21.15.92:0/33888621 conn(0x7f313809f7d0 msgr2=0x7f313809fbe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:57.059 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7f655f53e640 1 --2- 172.21.15.92:0/2323923172 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6558093cf0 0x7f6558094cd0 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7f6554009860 tx=0x7f655402a420 comp rx=0 tx=0).stop 2024-09-21T09:41:57.060 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7ffae8d53640 1 -- 172.21.15.92:0/1576089104 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ffae40933d0 msgr2=0x7ffae40937b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.060 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7ffae8d53640 1 --2- 172.21.15.92:0/1576089104 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ffae40933d0 0x7ffae40937b0 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7ffad4009900 tx=0x7ffad402a520 comp rx=0 tx=0).stop 2024-09-21T09:41:57.060 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7fe6306e7640 1 -- 172.21.15.92:0/984639091 shutdown_connections 2024-09-21T09:41:57.060 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7f313fd2c640 1 -- 172.21.15.92:0/33888621 shutdown_connections 2024-09-21T09:41:57.061 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7fe6306e7640 1 -- 172.21.15.92:0/984639091 wait complete. 2024-09-21T09:41:57.061 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7f313fd2c640 1 -- 172.21.15.92:0/33888621 wait complete. 2024-09-21T09:41:57.061 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7f655f53e640 1 -- 172.21.15.92:0/2323923172 shutdown_connections 2024-09-21T09:41:57.061 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7f655f53e640 1 --2- 172.21.15.92:0/2323923172 >> v2:172.21.15.92:3300/0 conn(0x7f65581302e0 0x7f65581326d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.062 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7f655f53e640 1 --2- 172.21.15.92:0/2323923172 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6558093cf0 0x7f6558094cd0 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.062 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.055+0000 7f655f53e640 1 -- 172.21.15.92:0/2323923172 >> 172.21.15.92:0/2323923172 conn(0x7f655809d580 msgr2=0x7f655809d990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:57.062 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.056+0000 7ffae8d53640 1 -- 172.21.15.92:0/1576089104 shutdown_connections 2024-09-21T09:41:57.062 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.056+0000 7ffae8d53640 1 --2- 172.21.15.92:0/1576089104 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ffae409a530 0x7ffae412a8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.063 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.056+0000 7ffae8d53640 1 --2- 172.21.15.92:0/1576089104 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ffae40933d0 0x7ffae40937b0 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.063 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.056+0000 7ffae8d53640 1 -- 172.21.15.92:0/1576089104 >> 172.21.15.92:0/1576089104 conn(0x7ffae409d6e0 msgr2=0x7ffae4099bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:57.063 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.056+0000 7f655f53e640 1 -- 172.21.15.92:0/2323923172 shutdown_connections 2024-09-21T09:41:57.063 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.056+0000 7f655f53e640 1 -- 172.21.15.92:0/2323923172 wait complete. 2024-09-21T09:41:57.064 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.056+0000 7ffae8d53640 1 -- 172.21.15.92:0/1576089104 shutdown_connections 2024-09-21T09:41:57.064 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.056+0000 7ffae8d53640 1 -- 172.21.15.92:0/1576089104 wait complete. 2024-09-21T09:41:57.064 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.056+0000 7f655f53e640 1 Processor -- start 2024-09-21T09:41:57.064 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7fe6306e7640 1 Processor -- start 2024-09-21T09:41:57.064 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f313fd2c640 1 Processor -- start 2024-09-21T09:41:57.065 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f655f53e640 1 -- start start 2024-09-21T09:41:57.065 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f313fd2c640 1 -- start start 2024-09-21T09:41:57.065 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7fe6306e7640 1 -- start start 2024-09-21T09:41:57.066 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7ffae8d53640 1 Processor -- start 2024-09-21T09:41:57.066 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7ffae8d53640 1 -- start start 2024-09-21T09:41:57.066 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f655f53e640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6558093cf0 0x7f6558125dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.066 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f655f53e640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6558095210 0x7f655812ee40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.066 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f655f53e640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f65581302e0 0x7f655812f380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.067 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7ffae8d53640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ffae40933d0 0x7ffae4070340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.067 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f313fd2c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f313808c320 0x7f313809d220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.067 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7fe6306e7640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe628095f10 0x7fe628127280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.067 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f655cad9640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6558095210 0x7f655812ee40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.067 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f655dadb640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f65581302e0 0x7f655812f380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.068 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f655cad9640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6558095210 0x7f655812ee40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:40940/0 (socket says 172.21.15.92:40940) 2024-09-21T09:41:57.068 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f655cad9640 1 -- 172.21.15.92:0/1340787815 learned_addr learned my addr 172.21.15.92:0/1340787815 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:57.068 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f313fd2c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f313808ce10 0x7f313809d760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.068 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f313fd2c640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f31381335f0 0x7f313808ec20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.068 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f313fd2c640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f3138136a70 con 0x7f313808ce10 2024-09-21T09:41:57.068 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f313fd2c640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f31381368f0 con 0x7f313808c320 2024-09-21T09:41:57.068 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7ffae8d53640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ffae409a530 0x7ffae4070880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.069 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f655dadb640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f65581302e0 0x7f655812f380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:46850/0 (socket says 172.21.15.92:46850) 2024-09-21T09:41:57.069 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f655d2da640 1 --2- 172.21.15.92:0/1340787815 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6558093cf0 0x7f6558125dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.069 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f313fd2c640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f3138136bf0 con 0x7f31381335f0 2024-09-21T09:41:57.069 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f655f53e640 1 -- 172.21.15.92:0/1340787815 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f6558133740 con 0x7f6558093cf0 2024-09-21T09:41:57.069 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f655f53e640 1 -- 172.21.15.92:0/1340787815 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f65581335c0 con 0x7f6558095210 2024-09-21T09:41:57.069 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f655f53e640 1 -- 172.21.15.92:0/1340787815 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f65581338c0 con 0x7f65581302e0 2024-09-21T09:41:57.070 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7ffae8d53640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ffae412ae10 0x7ffae4070dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.070 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7ffae259c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ffae40933d0 0x7ffae4070340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.070 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7ffae259c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ffae40933d0 0x7ffae4070340 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:40950/0 (socket says 172.21.15.92:40950) 2024-09-21T09:41:57.070 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f313dac8640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f313808c320 0x7f313809d220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.070 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f313dac8640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f313808c320 0x7f313809d220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:40966/0 (socket says 172.21.15.92:40966) 2024-09-21T09:41:57.070 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7f313dac8640 1 -- 172.21.15.92:0/3985339747 learned_addr learned my addr 172.21.15.92:0/3985339747 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:57.071 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7fe62e483640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe628095f10 0x7fe628127280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.071 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7fe62e483640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe628095f10 0x7fe628127280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:46858/0 (socket says 172.21.15.92:46858) 2024-09-21T09:41:57.071 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7fe62e483640 1 -- 172.21.15.92:0/3501963501 learned_addr learned my addr 172.21.15.92:0/3501963501 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:57.071 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7fe6306e7640 1 --2- 172.21.15.92:0/3501963501 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe628096830 0x7fe6281277c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.071 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.057+0000 7fe6306e7640 1 --2- 172.21.15.92:0/3501963501 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe62812bf60 0x7fe628127d00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.071 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7ffae8d53640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7ffae406c000 con 0x7ffae409a530 2024-09-21T09:41:57.072 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7ffae8d53640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7ffae406be80 con 0x7ffae40933d0 2024-09-21T09:41:57.072 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7fe6306e7640 1 -- 172.21.15.92:0/3501963501 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fe628134830 con 0x7fe628096830 2024-09-21T09:41:57.072 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7fe6306e7640 1 -- 172.21.15.92:0/3501963501 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fe6281346b0 con 0x7fe62812bf60 2024-09-21T09:41:57.072 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7fe6306e7640 1 -- 172.21.15.92:0/3501963501 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fe6281349b0 con 0x7fe628095f10 2024-09-21T09:41:57.072 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7fe62dc82640 1 --2- 172.21.15.92:0/3501963501 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe628096830 0x7fe6281277c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.072 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7fe62ec84640 1 --2- 172.21.15.92:0/3501963501 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe62812bf60 0x7fe628127d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.072 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7ffae8d53640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7ffae406c180 con 0x7ffae412ae10 2024-09-21T09:41:57.073 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7f655cad9640 1 -- 172.21.15.92:0/1340787815 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f65581302e0 msgr2=0x7f655812f380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.073 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7f655cad9640 1 --2- 172.21.15.92:0/1340787815 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f65581302e0 0x7f655812f380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.073 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7f655cad9640 1 -- 172.21.15.92:0/1340787815 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6558093cf0 msgr2=0x7f6558125dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.073 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7f655cad9640 1 --2- 172.21.15.92:0/1340787815 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6558093cf0 0x7f6558125dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.073 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7f655cad9640 1 -- 172.21.15.92:0/1340787815 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6558073950 con 0x7f6558095210 2024-09-21T09:41:57.073 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7f655cad9640 1 --2- 172.21.15.92:0/1340787815 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6558095210 0x7f655812ee40 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f654800c450 tx=0x7f654800c910 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:57.074 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7ffae259c640 1 -- 172.21.15.92:0/1412027883 learned_addr learned my addr 172.21.15.92:0/1412027883 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:57.074 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7ffae1d9b640 1 --2- 172.21.15.92:0/1412027883 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ffae409a530 0x7ffae4070880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.074 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7ffae2d9d640 1 --2- 172.21.15.92:0/1412027883 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ffae412ae10 0x7ffae4070dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.074 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7f654dffb640 1 -- 172.21.15.92:0/1340787815 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f65480052d0 con 0x7f6558095210 2024-09-21T09:41:57.074 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7ffae259c640 1 -- 172.21.15.92:0/1412027883 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ffae412ae10 msgr2=0x7ffae4070dc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.075 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7fe62e483640 1 -- 172.21.15.92:0/3501963501 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe62812bf60 msgr2=0x7fe628127d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.075 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7f654dffb640 1 -- 172.21.15.92:0/1340787815 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f6548005470 con 0x7f6558095210 2024-09-21T09:41:57.075 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7f313e2c9640 1 --2- 172.21.15.92:0/3985339747 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f31381335f0 0x7f313808ec20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.075 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7f313d2c7640 1 --2- 172.21.15.92:0/3985339747 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f313808ce10 0x7f313809d760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.075 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7f655f53e640 1 -- 172.21.15.92:0/1340787815 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f655806be60 con 0x7f6558095210 2024-09-21T09:41:57.076 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7f313dac8640 1 -- 172.21.15.92:0/3985339747 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f31381335f0 msgr2=0x7f313808ec20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.076 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7ffae259c640 1 --2- 172.21.15.92:0/1412027883 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ffae412ae10 0x7ffae4070dc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.076 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7ffae259c640 1 -- 172.21.15.92:0/1412027883 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ffae409a530 msgr2=0x7ffae4070880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.076 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7f654dffb640 1 -- 172.21.15.92:0/1340787815 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6548012890 con 0x7f6558095210 2024-09-21T09:41:57.077 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7f313dac8640 1 --2- 172.21.15.92:0/3985339747 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f31381335f0 0x7f313808ec20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.077 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7f313dac8640 1 -- 172.21.15.92:0/3985339747 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f313808ce10 msgr2=0x7f313809d760 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.077 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7f313dac8640 1 --2- 172.21.15.92:0/3985339747 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f313808ce10 0x7f313809d760 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.077 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7f313dac8640 1 -- 172.21.15.92:0/3985339747 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3138090660 con 0x7f313808c320 2024-09-21T09:41:57.077 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7fe62e483640 1 --2- 172.21.15.92:0/3501963501 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe62812bf60 0x7fe628127d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.077 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7fe62e483640 1 -- 172.21.15.92:0/3501963501 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe628096830 msgr2=0x7fe6281277c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.077 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7fe62e483640 1 --2- 172.21.15.92:0/3501963501 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe628096830 0x7fe6281277c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.077 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7fe62e483640 1 -- 172.21.15.92:0/3501963501 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe628126360 con 0x7fe628095f10 2024-09-21T09:41:57.078 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7ffae259c640 1 --2- 172.21.15.92:0/1412027883 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ffae409a530 0x7ffae4070880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.078 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7ffae259c640 1 -- 172.21.15.92:0/1412027883 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ffae4071e70 con 0x7ffae40933d0 2024-09-21T09:41:57.078 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7f313e2c9640 1 --2- 172.21.15.92:0/3985339747 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f31381335f0 0x7f313808ec20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T09:41:57.078 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.058+0000 7f313d2c7640 1 --2- 172.21.15.92:0/3985339747 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f313808ce10 0x7f313809d760 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:41:57.079 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.059+0000 7fe62e483640 1 --2- 172.21.15.92:0/3501963501 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe628095f10 0x7fe628127280 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fe6240099f0 tx=0x7fe624032640 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:57.079 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.059+0000 7ffae259c640 1 --2- 172.21.15.92:0/1412027883 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ffae40933d0 0x7ffae4070340 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7ffad4009850 tx=0x7ffad40326c0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:57.079 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.059+0000 7f313dac8640 1 --2- 172.21.15.92:0/3985339747 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f313808c320 0x7f313809d220 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f313400c430 tx=0x7f313400c8f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:57.080 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.059+0000 7f655f53e640 1 -- 172.21.15.92:0/1340787815 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f6558122870 con 0x7f6558095210 2024-09-21T09:41:57.080 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.059+0000 7ffad2ffd640 1 -- 172.21.15.92:0/1412027883 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffad403f070 con 0x7ffae40933d0 2024-09-21T09:41:57.080 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.059+0000 7f312e7fc640 1 -- 172.21.15.92:0/3985339747 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f31340052d0 con 0x7f313808c320 2024-09-21T09:41:57.080 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.059+0000 7ffae8d53640 1 -- 172.21.15.92:0/1412027883 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ffae4072f50 con 0x7ffae40933d0 2024-09-21T09:41:57.081 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.059+0000 7ffad2ffd640 1 -- 172.21.15.92:0/1412027883 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7ffad4004460 con 0x7ffae40933d0 2024-09-21T09:41:57.081 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.059+0000 7f313fd2c640 1 -- 172.21.15.92:0/3985339747 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f313809dca0 con 0x7f313808c320 2024-09-21T09:41:57.081 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.059+0000 7ffad2ffd640 1 -- 172.21.15.92:0/1412027883 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ffad4035030 con 0x7ffae40933d0 2024-09-21T09:41:57.081 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.059+0000 7f655f53e640 1 -- 172.21.15.92:0/1340787815 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f6558122ad0 con 0x7f6558095210 2024-09-21T09:41:57.082 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.059+0000 7f313fd2c640 1 -- 172.21.15.92:0/3985339747 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f3138091740 con 0x7f313808c320 2024-09-21T09:41:57.082 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.059+0000 7fe61effd640 1 -- 172.21.15.92:0/3501963501 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe62403f070 con 0x7fe628095f10 2024-09-21T09:41:57.082 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.059+0000 7ffae8d53640 1 -- 172.21.15.92:0/1412027883 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7ffae41c3870 con 0x7ffae40933d0 2024-09-21T09:41:57.082 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.059+0000 7fe61effd640 1 -- 172.21.15.92:0/3501963501 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fe624004460 con 0x7fe628095f10 2024-09-21T09:41:57.083 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.059+0000 7fe6306e7640 1 -- 172.21.15.92:0/3501963501 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe628128d50 con 0x7fe628095f10 2024-09-21T09:41:57.083 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.060+0000 7f313fd2c640 1 -- 172.21.15.92:0/3985339747 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f3138090f20 con 0x7f313808c320 2024-09-21T09:41:57.083 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.060+0000 7fe61effd640 1 -- 172.21.15.92:0/3501963501 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe624035030 con 0x7fe628095f10 2024-09-21T09:41:57.083 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.060+0000 7fe6306e7640 1 -- 172.21.15.92:0/3501963501 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fe628093950 con 0x7fe628095f10 2024-09-21T09:41:57.083 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.060+0000 7ffae8d53640 1 -- 172.21.15.92:0/1412027883 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_get_version(what=osdmap handle=1) -- 0x7ffae4124890 con 0x7ffae40933d0 2024-09-21T09:41:57.084 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.060+0000 7f312e7fc640 1 -- 172.21.15.92:0/3985339747 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f3134005470 con 0x7f313808c320 2024-09-21T09:41:57.084 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.061+0000 7f312e7fc640 1 -- 172.21.15.92:0/3985339747 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3134020060 con 0x7f313808c320 2024-09-21T09:41:57.084 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.060+0000 7f654dffb640 1 -- 172.21.15.92:0/1340787815 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f65480035f0 con 0x7f6558095210 2024-09-21T09:41:57.084 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.061+0000 7f654dffb640 1 -- 172.21.15.92:0/1340787815 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7f654803be20 con 0x7f6558095210 2024-09-21T09:41:57.084 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.061+0000 7f654dffb640 1 -- 172.21.15.92:0/1340787815 --> v1:172.21.15.92:6813/1359151798 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f654800bf30 con 0x7f653c04e680 2024-09-21T09:41:57.085 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.061+0000 7fe6306e7640 1 -- 172.21.15.92:0/3501963501 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_get_version(what=osdmap handle=1) -- 0x7fe628093bd0 con 0x7fe628095f10 2024-09-21T09:41:57.085 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.061+0000 7fe61effd640 1 -- 172.21.15.92:0/3501963501 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fe624042070 con 0x7fe628095f10 2024-09-21T09:41:57.085 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.061+0000 7f312e7fc640 1 -- 172.21.15.92:0/3985339747 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f3134007300 con 0x7f313808c320 2024-09-21T09:41:57.085 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.062+0000 7f654dffb640 1 -- 172.21.15.92:0/1340787815 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_get_version_reply(handle=1 version=18) ==== 24+0+0 (secure 0 0 0) 0x7f654803a020 con 0x7f6558095210 2024-09-21T09:41:57.085 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.062+0000 7ffad2ffd640 1 -- 172.21.15.92:0/1412027883 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7ffad4042070 con 0x7ffae40933d0 2024-09-21T09:41:57.086 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.062+0000 7fe61effd640 1 -- 172.21.15.92:0/3501963501 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7fe62407eda0 con 0x7fe628095f10 2024-09-21T09:41:57.086 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.062+0000 7f312e7fc640 1 -- 172.21.15.92:0/3985339747 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7f313405df00 con 0x7f313808c320 2024-09-21T09:41:57.086 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.062+0000 7f312e7fc640 1 -- 172.21.15.92:0/3985339747 --> v1:172.21.15.92:6809/296401696 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f311c052340 con 0x7f311c04e650 2024-09-21T09:41:57.086 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.062+0000 7f312e7fc640 1 -- 172.21.15.92:0/3985339747 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_get_version_reply(handle=1 version=18) ==== 24+0+0 (secure 0 0 0) 0x7f313405e2f0 con 0x7f313808c320 2024-09-21T09:41:57.087 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.062+0000 7fe61effd640 1 -- 172.21.15.92:0/3501963501 --> v1:172.21.15.92:6801/1829519295 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7fe624049e50 con 0x7fe60c04e790 2024-09-21T09:41:57.087 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.062+0000 7fe61effd640 1 -- 172.21.15.92:0/3501963501 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_get_version_reply(handle=1 version=18) ==== 24+0+0 (secure 0 0 0) 0x7fe624083030 con 0x7fe628095f10 2024-09-21T09:41:57.087 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.062+0000 7ffad2ffd640 1 -- 172.21.15.92:0/1412027883 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7ffad407f000 con 0x7ffae40933d0 2024-09-21T09:41:57.087 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.062+0000 7ffad2ffd640 1 -- 172.21.15.92:0/1412027883 --> v1:172.21.15.92:6805/195102377 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7ffac40523b0 con 0x7ffac404e6c0 2024-09-21T09:41:57.087 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.062+0000 7ffad2ffd640 1 -- 172.21.15.92:0/1412027883 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_get_version_reply(handle=1 version=18) ==== 24+0+0 (secure 0 0 0) 0x7ffad407f3f0 con 0x7ffae40933d0 2024-09-21T09:41:57.087 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.063+0000 7f654dffb640 1 -- 172.21.15.92:0/1340787815 <== osd.1 v1:172.21.15.92:6813/1359151798 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f654800bf30 con 0x7f653c04e680 2024-09-21T09:41:57.088 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.063+0000 7ffad2ffd640 1 -- 172.21.15.92:0/1412027883 <== osd.0 v1:172.21.15.92:6805/195102377 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7ffac40523b0 con 0x7ffac404e6c0 2024-09-21T09:41:57.088 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.063+0000 7f312e7fc640 1 -- 172.21.15.92:0/3985339747 <== osd.2 v1:172.21.15.92:6809/296401696 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f311c052340 con 0x7f311c04e650 2024-09-21T09:41:57.088 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.064+0000 7fe61effd640 1 -- 172.21.15.92:0/3501963501 <== osd.3 v1:172.21.15.92:6801/1829519295 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7fe624049e50 con 0x7fe60c04e790 2024-09-21T09:41:57.089 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.077+0000 7ffae8d53640 1 -- 172.21.15.92:0/1412027883 --> v1:172.21.15.92:6805/195102377 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7ffae4124ae0 con 0x7ffac404e6c0 2024-09-21T09:41:57.089 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.077+0000 7f655f53e640 1 -- 172.21.15.92:0/1340787815 --> v1:172.21.15.92:6813/1359151798 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f655812fb40 con 0x7f653c04e680 2024-09-21T09:41:57.089 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.077+0000 7ffad2ffd640 1 -- 172.21.15.92:0/1412027883 <== osd.0 v1:172.21.15.92:6805/195102377 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 3325991546) 0x7ffae4124ae0 con 0x7ffac404e6c0 2024-09-21T09:41:57.089 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.077+0000 7fe6306e7640 1 -- 172.21.15.92:0/3501963501 --> v1:172.21.15.92:6801/1829519295 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7fe628093d80 con 0x7fe60c04e790 2024-09-21T09:41:57.090 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.077+0000 7f654dffb640 1 -- 172.21.15.92:0/1340787815 <== osd.1 v1:172.21.15.92:6813/1359151798 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 654699149) 0x7f655812fb40 con 0x7f653c04e680 2024-09-21T09:41:57.090 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.077+0000 7ffae8d53640 1 -- 172.21.15.92:0/1412027883 >> v1:172.21.15.92:6805/195102377 conn(0x7ffac404e6c0 legacy=0x7ffac4050b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.090 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.077+0000 7f313fd2c640 1 -- 172.21.15.92:0/3985339747 --> v1:172.21.15.92:6809/296401696 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f3138092340 con 0x7f311c04e650 2024-09-21T09:41:57.091 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.077+0000 7fe61effd640 1 -- 172.21.15.92:0/3501963501 <== osd.3 v1:172.21.15.92:6801/1829519295 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 675830894) 0x7fe628093d80 con 0x7fe60c04e790 2024-09-21T09:41:57.091 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.077+0000 7ffae8d53640 1 -- 172.21.15.92:0/1412027883 >> v1:172.21.15.92:6800/1280404317 conn(0x7ffac4047fd0 legacy=0x7ffac404a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.091 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.077+0000 7ffae8d53640 1 -- 172.21.15.92:0/1412027883 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ffae40933d0 msgr2=0x7ffae4070340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.092 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.077+0000 7f312e7fc640 1 -- 172.21.15.92:0/3985339747 <== osd.2 v1:172.21.15.92:6809/296401696 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 3325991546) 0x7f3138092340 con 0x7f311c04e650 2024-09-21T09:41:57.092 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.077+0000 7f655f53e640 1 -- 172.21.15.92:0/1340787815 >> v1:172.21.15.92:6813/1359151798 conn(0x7f653c04e680 legacy=0x7f653c050b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.092 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.077+0000 7ffae8d53640 1 --2- 172.21.15.92:0/1412027883 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ffae40933d0 0x7ffae4070340 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7ffad4009850 tx=0x7ffad40326c0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.092 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7ffae2d9d640 1 -- 172.21.15.92:0/1412027883 reap_dead start 2024-09-21T09:41:57.093 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7ffae8d53640 1 -- 172.21.15.92:0/1412027883 shutdown_connections 2024-09-21T09:41:57.093 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7ffae8d53640 1 -- 172.21.15.92:0/1412027883 >> 172.21.15.92:0/1412027883 conn(0x7ffae409d6e0 msgr2=0x7ffae4124180 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:57.093 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7f655f53e640 1 -- 172.21.15.92:0/1340787815 >> v1:172.21.15.92:6800/1280404317 conn(0x7f653c048000 legacy=0x7f653c04a4c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.093 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7f655f53e640 1 -- 172.21.15.92:0/1340787815 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6558095210 msgr2=0x7f655812ee40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.093 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7f655f53e640 1 --2- 172.21.15.92:0/1340787815 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6558095210 0x7f655812ee40 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f654800c450 tx=0x7f654800c910 comp rx=0 tx=0).stop 2024-09-21T09:41:57.094 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7fe6306e7640 1 -- 172.21.15.92:0/3501963501 >> v1:172.21.15.92:6801/1829519295 conn(0x7fe60c04e790 legacy=0x7fe60c050c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.094 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7f655dadb640 1 -- 172.21.15.92:0/1340787815 reap_dead start 2024-09-21T09:41:57.094 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7ffae8d53640 1 -- 172.21.15.92:0/1412027883 shutdown_connections 2024-09-21T09:41:57.094 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7ffae8d53640 1 -- 172.21.15.92:0/1412027883 wait complete. 2024-09-21T09:41:57.095 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7f313fd2c640 1 -- 172.21.15.92:0/3985339747 >> v1:172.21.15.92:6809/296401696 conn(0x7f311c04e650 legacy=0x7f311c050ad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.095 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7f655f53e640 1 -- 172.21.15.92:0/1340787815 shutdown_connections 2024-09-21T09:41:57.095 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7f655f53e640 1 -- 172.21.15.92:0/1340787815 >> 172.21.15.92:0/1340787815 conn(0x7f655809d580 msgr2=0x7f6558131d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:57.095 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7f655f53e640 1 -- 172.21.15.92:0/1340787815 shutdown_connections 2024-09-21T09:41:57.095 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7f313fd2c640 1 -- 172.21.15.92:0/3985339747 >> v1:172.21.15.92:6800/1280404317 conn(0x7f311c047fd0 legacy=0x7f311c04a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.096 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7fe6306e7640 1 -- 172.21.15.92:0/3501963501 >> v1:172.21.15.92:6800/1280404317 conn(0x7fe60c0480a0 legacy=0x7fe60c04a560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.096 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7f655f53e640 1 -- 172.21.15.92:0/1340787815 wait complete. 2024-09-21T09:41:57.096 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7fe6306e7640 1 -- 172.21.15.92:0/3501963501 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe628095f10 msgr2=0x7fe628127280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.096 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7fe6306e7640 1 --2- 172.21.15.92:0/3501963501 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe628095f10 0x7fe628127280 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fe6240099f0 tx=0x7fe624032640 comp rx=0 tx=0).stop 2024-09-21T09:41:57.096 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7fe62ec84640 1 -- 172.21.15.92:0/3501963501 reap_dead start 2024-09-21T09:41:57.097 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7fe6306e7640 1 -- 172.21.15.92:0/3501963501 shutdown_connections 2024-09-21T09:41:57.097 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7fe6306e7640 1 -- 172.21.15.92:0/3501963501 >> 172.21.15.92:0/3501963501 conn(0x7fe62809c590 msgr2=0x7fe6280998a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:57.097 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7f313fd2c640 1 -- 172.21.15.92:0/3985339747 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f313808c320 msgr2=0x7f313809d220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.097 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7f313fd2c640 1 --2- 172.21.15.92:0/3985339747 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f313808c320 0x7f313809d220 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f313400c430 tx=0x7f313400c8f0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.097 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7f313e2c9640 1 -- 172.21.15.92:0/3985339747 reap_dead start 2024-09-21T09:41:57.097 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7f313fd2c640 1 -- 172.21.15.92:0/3985339747 shutdown_connections 2024-09-21T09:41:57.098 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7f313fd2c640 1 -- 172.21.15.92:0/3985339747 >> 172.21.15.92:0/3985339747 conn(0x7f313809f7d0 msgr2=0x7f31381276b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:57.098 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7f313fd2c640 1 -- 172.21.15.92:0/3985339747 shutdown_connections 2024-09-21T09:41:57.098 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7fe6306e7640 1 -- 172.21.15.92:0/3501963501 shutdown_connections 2024-09-21T09:41:57.098 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7fe6306e7640 1 -- 172.21.15.92:0/3501963501 wait complete. 2024-09-21T09:41:57.098 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.078+0000 7f313fd2c640 1 -- 172.21.15.92:0/3985339747 wait complete. 2024-09-21T09:41:57.099 INFO:teuthology.orchestra.run.smithi092.stdout:47244640262 2024-09-21T09:41:57.099 INFO:teuthology.orchestra.run.smithi092.stdout:38654705669 2024-09-21T09:41:57.100 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T09:41:57.100 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T09:41:57.101 INFO:teuthology.orchestra.run.smithi092.stdout:47244640262 2024-09-21T09:41:57.101 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T09:41:57.102 INFO:teuthology.orchestra.run.smithi092.stdout:47244640260 2024-09-21T09:41:57.103 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T09:41:57.246 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.244+0000 7fa6bcef0640 1 -- 172.21.15.92:0/2669625625 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fa6b8131660 msgr2=0x7fa6b8133a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.246 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.244+0000 7fa6bcef0640 1 --2- 172.21.15.92:0/2669625625 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fa6b8131660 0x7fa6b8133a50 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7fa6b0005fd0 tx=0x7fa6b002a780 comp rx=0 tx=0).stop 2024-09-21T09:41:57.246 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.244+0000 7fc98038a640 1 -- 172.21.15.92:0/907994707 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc97812af60 msgr2=0x7fc97812b340 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.246 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.244+0000 7fc98038a640 1 --2- 172.21.15.92:0/907994707 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc97812af60 0x7fc97812b340 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7fc974009900 tx=0x7fc97402a580 comp rx=0 tx=0).stop 2024-09-21T09:41:57.247 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.244+0000 7fa6bcef0640 1 -- 172.21.15.92:0/2669625625 shutdown_connections 2024-09-21T09:41:57.247 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.244+0000 7fa6bcef0640 1 --2- 172.21.15.92:0/2669625625 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fa6b8131660 0x7fa6b8133a50 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.247 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.244+0000 7fa6bcef0640 1 --2- 172.21.15.92:0/2669625625 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fa6b812ed30 0x7fa6b8131120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.247 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.244+0000 7fa6bcef0640 1 -- 172.21.15.92:0/2669625625 >> 172.21.15.92:0/2669625625 conn(0x7fa6b809c590 msgr2=0x7fa6b809c9a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:57.247 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.244+0000 7fa6bcef0640 1 -- 172.21.15.92:0/2669625625 shutdown_connections 2024-09-21T09:41:57.247 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.244+0000 7fc98038a640 1 -- 172.21.15.92:0/907994707 shutdown_connections 2024-09-21T09:41:57.247 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.244+0000 7fc98038a640 1 --2- 172.21.15.92:0/907994707 >> v2:172.21.15.92:3300/0 conn(0x7fc97812b880 0x7fc9780960b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.244+0000 7fc98038a640 1 --2- 172.21.15.92:0/907994707 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc97812af60 0x7fc97812b340 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.244+0000 7fa6bcef0640 1 -- 172.21.15.92:0/2669625625 wait complete. 2024-09-21T09:41:57.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.244+0000 7fc98038a640 1 -- 172.21.15.92:0/907994707 >> 172.21.15.92:0/907994707 conn(0x7fc97809ac70 msgr2=0x7fc97809b080 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:57.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.244+0000 7fc98038a640 1 -- 172.21.15.92:0/907994707 shutdown_connections 2024-09-21T09:41:57.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.244+0000 7fc98038a640 1 -- 172.21.15.92:0/907994707 wait complete. 2024-09-21T09:41:57.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.244+0000 7fdc5ce24640 1 -- 172.21.15.92:0/871685012 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fdc5812b230 msgr2=0x7fdc58132e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.245+0000 7fdc5ce24640 1 --2- 172.21.15.92:0/871685012 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fdc5812b230 0x7fdc58132e60 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fdc50005fd0 tx=0x7fdc5002a980 comp rx=0 tx=0).stop 2024-09-21T09:41:57.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.245+0000 7fdc5ce24640 1 -- 172.21.15.92:0/871685012 shutdown_connections 2024-09-21T09:41:57.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.245+0000 7fdc5ce24640 1 --2- 172.21.15.92:0/871685012 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fdc5812b230 0x7fdc58132e60 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.245+0000 7fdc5ce24640 1 --2- 172.21.15.92:0/871685012 >> v2:172.21.15.92:3300/0 conn(0x7fdc580963e0 0x7fdc580967c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.245+0000 7fdc5ce24640 1 -- 172.21.15.92:0/871685012 >> 172.21.15.92:0/871685012 conn(0x7fdc5809cd80 msgr2=0x7fdc5809d190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:57.250 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.245+0000 7fdc5ce24640 1 -- 172.21.15.92:0/871685012 shutdown_connections 2024-09-21T09:41:57.250 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.245+0000 7fdc5ce24640 1 -- 172.21.15.92:0/871685012 wait complete. 2024-09-21T09:41:57.250 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.246+0000 7fdc5ce24640 1 Processor -- start 2024-09-21T09:41:57.250 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.246+0000 7fa6bcef0640 1 Processor -- start 2024-09-21T09:41:57.250 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.246+0000 7fdc5ce24640 1 -- start start 2024-09-21T09:41:57.250 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.246+0000 7fa6bcef0640 1 -- start start 2024-09-21T09:41:57.251 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.246+0000 7fc98038a640 1 Processor -- start 2024-09-21T09:41:57.251 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.246+0000 7fc98038a640 1 -- start start 2024-09-21T09:41:57.252 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.246+0000 7fdc5ce24640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fdc58070b10 0x7fdc58122150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.252 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.246+0000 7fdc5ce24640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fdc580963e0 0x7fdc58122690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.252 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.246+0000 7fdc5ce24640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fdc5812b230 0x7fdc58122bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.252 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.246+0000 7fdc5ce24640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fdc58133840 con 0x7fdc580963e0 2024-09-21T09:41:57.252 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.246+0000 7fdc5ce24640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fdc581336c0 con 0x7fdc58070b10 2024-09-21T09:41:57.252 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.246+0000 7fdc5ce24640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fdc581339c0 con 0x7fdc5812b230 2024-09-21T09:41:57.252 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fdc5659c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fdc58070b10 0x7fdc58122150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.252 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fdc55d9b640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fdc580963e0 0x7fdc58122690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.252 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fdc55d9b640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fdc580963e0 0x7fdc58122690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:40388/0 (socket says 172.21.15.92:40388) 2024-09-21T09:41:57.252 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fdc55d9b640 1 -- 172.21.15.92:0/1693909738 learned_addr learned my addr 172.21.15.92:0/1693909738 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:57.253 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fa6bcef0640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fa6b8095f10 0x7fa6b8129820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.253 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fdc56d9d640 1 --2- 172.21.15.92:0/1693909738 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fdc5812b230 0x7fdc58122bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.253 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fc98038a640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc9780965f0 0x7fc978121ec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.254 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fa6bcef0640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fa6b812ed30 0x7fa6b8129d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.254 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fa6b659c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fa6b8095f10 0x7fa6b8129820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.254 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fa6b659c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fa6b8095f10 0x7fa6b8129820 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:41010/0 (socket says 172.21.15.92:41010) 2024-09-21T09:41:57.254 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fc98038a640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc97812af60 0x7fc978122400 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.254 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fdc55d9b640 1 -- 172.21.15.92:0/1693909738 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fdc5812b230 msgr2=0x7fdc58122bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.254 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fdc55d9b640 1 --2- 172.21.15.92:0/1693909738 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fdc5812b230 0x7fdc58122bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.255 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fdc55d9b640 1 -- 172.21.15.92:0/1693909738 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fdc58070b10 msgr2=0x7fdc58122150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.255 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fdc55d9b640 1 --2- 172.21.15.92:0/1693909738 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fdc58070b10 0x7fdc58122150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.255 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fdc55d9b640 1 -- 172.21.15.92:0/1693909738 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdc5812a840 con 0x7fdc580963e0 2024-09-21T09:41:57.255 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fa6b659c640 1 -- 172.21.15.92:0/2782163908 learned_addr learned my addr 172.21.15.92:0/2782163908 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:57.255 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fa6bcef0640 1 --2- 172.21.15.92:0/2782163908 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fa6b8131660 0x7fa6b812a2a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.255 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fa6b5d9b640 1 --2- 172.21.15.92:0/2782163908 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fa6b812ed30 0x7fa6b8129d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.256 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fdc55d9b640 1 --2- 172.21.15.92:0/1693909738 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fdc580963e0 0x7fdc58122690 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7fdc4c00c450 tx=0x7fdc4c00c910 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:57.256 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fc97d925640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc97812af60 0x7fc978122400 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.256 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fc97d925640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc97812af60 0x7fc978122400 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:40406/0 (socket says 172.21.15.92:40406) 2024-09-21T09:41:57.256 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fa6b6d9d640 1 --2- 172.21.15.92:0/2782163908 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fa6b8131660 0x7fa6b812a2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.256 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fc98038a640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc97812b880 0x7fc978122940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.257 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fa6bcef0640 1 -- 172.21.15.92:0/2782163908 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fa6b81347b0 con 0x7fa6b812ed30 2024-09-21T09:41:57.257 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fc97e126640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc9780965f0 0x7fc978121ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.257 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fc97e126640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc9780965f0 0x7fc978121ec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:46894/0 (socket says 172.21.15.92:46894) 2024-09-21T09:41:57.257 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fa6bcef0640 1 -- 172.21.15.92:0/2782163908 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fa6b8134630 con 0x7fa6b8095f10 2024-09-21T09:41:57.258 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fdc46ffd640 1 -- 172.21.15.92:0/1693909738 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdc4c0052d0 con 0x7fdc580963e0 2024-09-21T09:41:57.258 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fa6bcef0640 1 -- 172.21.15.92:0/2782163908 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fa6b8134930 con 0x7fa6b8131660 2024-09-21T09:41:57.258 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fdc46ffd640 1 -- 172.21.15.92:0/1693909738 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fdc4c005470 con 0x7fdc580963e0 2024-09-21T09:41:57.258 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.247+0000 7fdc5ce24640 1 -- 172.21.15.92:0/1693909738 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fdc581245a0 con 0x7fdc580963e0 2024-09-21T09:41:57.259 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.248+0000 7fc97e927640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc97812b880 0x7fc978122940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.259 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.248+0000 7fc97e927640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc97812b880 0x7fc978122940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:41016/0 (socket says 172.21.15.92:41016) 2024-09-21T09:41:57.259 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.248+0000 7fc97d925640 1 -- 172.21.15.92:0/3961855085 learned_addr learned my addr 172.21.15.92:0/3961855085 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:57.259 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.248+0000 7fdc5ce24640 1 -- 172.21.15.92:0/1693909738 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fdc58124da0 con 0x7fdc580963e0 2024-09-21T09:41:57.260 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.248+0000 7fc98038a640 1 -- 172.21.15.92:0/3961855085 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fc978133d20 con 0x7fc97812af60 2024-09-21T09:41:57.260 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.248+0000 7fc98038a640 1 -- 172.21.15.92:0/3961855085 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fc978133ba0 con 0x7fc97812b880 2024-09-21T09:41:57.260 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.248+0000 7fc98038a640 1 -- 172.21.15.92:0/3961855085 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fc978133ea0 con 0x7fc9780965f0 2024-09-21T09:41:57.260 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.248+0000 7fdc46ffd640 1 -- 172.21.15.92:0/1693909738 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fdc4c012870 con 0x7fdc580963e0 2024-09-21T09:41:57.261 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.248+0000 7fa6b5d9b640 1 -- 172.21.15.92:0/2782163908 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fa6b8131660 msgr2=0x7fa6b812a2a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.261 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.248+0000 7fa6b5d9b640 1 --2- 172.21.15.92:0/2782163908 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fa6b8131660 0x7fa6b812a2a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.261 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.248+0000 7fa6b5d9b640 1 -- 172.21.15.92:0/2782163908 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fa6b8095f10 msgr2=0x7fa6b8129820 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.261 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.248+0000 7fa6b5d9b640 1 --2- 172.21.15.92:0/2782163908 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fa6b8095f10 0x7fa6b8129820 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.261 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.248+0000 7fa6b5d9b640 1 -- 172.21.15.92:0/2782163908 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa6b81285b0 con 0x7fa6b812ed30 2024-09-21T09:41:57.261 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.248+0000 7fa6b6d9d640 1 --2- 172.21.15.92:0/2782163908 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fa6b8131660 0x7fa6b812a2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:57.261 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.248+0000 7fa6b659c640 1 --2- 172.21.15.92:0/2782163908 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fa6b8095f10 0x7fa6b8129820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:41:57.261 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.248+0000 7fa6b5d9b640 1 --2- 172.21.15.92:0/2782163908 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fa6b812ed30 0x7fa6b8129d60 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7fa6ac00d740 tx=0x7fa6ac00dc00 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:57.262 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.249+0000 7fc97d925640 1 -- 172.21.15.92:0/3961855085 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc9780965f0 msgr2=0x7fc978121ec0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.262 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.249+0000 7fc97d925640 1 --2- 172.21.15.92:0/3961855085 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc9780965f0 0x7fc978121ec0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.262 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.249+0000 7fc97d925640 1 -- 172.21.15.92:0/3961855085 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc97812b880 msgr2=0x7fc978122940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.262 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.249+0000 7fc97d925640 1 --2- 172.21.15.92:0/3961855085 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc97812b880 0x7fc978122940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.262 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.249+0000 7fc97d925640 1 -- 172.21.15.92:0/3961855085 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc978123d70 con 0x7fc97812af60 2024-09-21T09:41:57.262 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.249+0000 7fc97e126640 1 --2- 172.21.15.92:0/3961855085 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc9780965f0 0x7fc978121ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T09:41:57.262 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.249+0000 7fc97e927640 1 --2- 172.21.15.92:0/3961855085 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc97812b880 0x7fc978122940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T09:41:57.262 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.249+0000 7fa6a6ffd640 1 -- 172.21.15.92:0/2782163908 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6ac017070 con 0x7fa6b812ed30 2024-09-21T09:41:57.263 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.249+0000 7fc97d925640 1 --2- 172.21.15.92:0/3961855085 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc97812af60 0x7fc978122400 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7fc96800d950 tx=0x7fc96800de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:57.263 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.249+0000 7fdc46ffd640 1 -- 172.21.15.92:0/1693909738 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fdc4c012a10 con 0x7fdc580963e0 2024-09-21T09:41:57.263 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.249+0000 7fa6a6ffd640 1 -- 172.21.15.92:0/2782163908 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fa6ac00de50 con 0x7fa6b812ed30 2024-09-21T09:41:57.263 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.249+0000 7fa6a6ffd640 1 -- 172.21.15.92:0/2782163908 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa6ac010460 con 0x7fa6b812ed30 2024-09-21T09:41:57.263 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.249+0000 7fa6bcef0640 1 -- 172.21.15.92:0/2782163908 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa6b8126360 con 0x7fa6b812ed30 2024-09-21T09:41:57.263 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.249+0000 7fa6bcef0640 1 -- 172.21.15.92:0/2782163908 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa6b812b180 con 0x7fa6b812ed30 2024-09-21T09:41:57.264 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.249+0000 7fc96effd640 1 -- 172.21.15.92:0/3961855085 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc968017070 con 0x7fc97812af60 2024-09-21T09:41:57.264 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.249+0000 7fc96effd640 1 -- 172.21.15.92:0/3961855085 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fc968009d20 con 0x7fc97812af60 2024-09-21T09:41:57.264 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.249+0000 7fc96effd640 1 -- 172.21.15.92:0/3961855085 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc9680106c0 con 0x7fc97812af60 2024-09-21T09:41:57.264 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.249+0000 7fdc5ce24640 1 -- 172.21.15.92:0/1693909738 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fdc3c005290 con 0x7fdc580963e0 2024-09-21T09:41:57.264 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.249+0000 7fc98038a640 1 -- 172.21.15.92:0/3961855085 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc978072b80 con 0x7fc97812af60 2024-09-21T09:41:57.265 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.249+0000 7fc98038a640 1 -- 172.21.15.92:0/3961855085 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc978124620 con 0x7fc97812af60 2024-09-21T09:41:57.265 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.250+0000 7fdc46ffd640 1 -- 172.21.15.92:0/1693909738 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7fdc4c05d3f0 con 0x7fdc580963e0 2024-09-21T09:41:57.265 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.250+0000 7fa6a6ffd640 1 -- 172.21.15.92:0/2782163908 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fa6ac01a070 con 0x7fa6b812ed30 2024-09-21T09:41:57.265 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.251+0000 7fc98038a640 1 -- 172.21.15.92:0/3961855085 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc97809daa0 con 0x7fc97812af60 2024-09-21T09:41:57.266 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.251+0000 7fa6a6ffd640 1 -- 172.21.15.92:0/2782163908 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7fa6ac05da00 con 0x7fa6b812ed30 2024-09-21T09:41:57.266 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.251+0000 7fa6bcef0640 1 -- 172.21.15.92:0/2782163908 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa6b812cc20 con 0x7fa6b812ed30 2024-09-21T09:41:57.266 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.251+0000 7fc96effd640 1 -- 172.21.15.92:0/3961855085 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fc968010860 con 0x7fc97812af60 2024-09-21T09:41:57.266 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.254+0000 7fdc46ffd640 1 -- 172.21.15.92:0/1693909738 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7fdc4c03c7b0 con 0x7fdc580963e0 2024-09-21T09:41:57.267 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.254+0000 7fc96effd640 1 -- 172.21.15.92:0/3961855085 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7fc96803d460 con 0x7fc97812af60 2024-09-21T09:41:57.267 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.258+0000 7fa6a6ffd640 1 -- 172.21.15.92:0/2782163908 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7fa6b812cc20 con 0x7fa6b812ed30 2024-09-21T09:41:57.267 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.265+0000 7fc96effd640 1 -- 172.21.15.92:0/3961855085 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7fc97809daa0 con 0x7fc97812af60 2024-09-21T09:41:57.290 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.288+0000 7f45945fd640 1 -- 172.21.15.92:0/2647327299 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f458c0971e0 msgr2=0x7f458c131360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.290 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.288+0000 7f45945fd640 1 --2- 172.21.15.92:0/2647327299 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f458c0971e0 0x7f458c131360 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f457c009830 tx=0x7f457c02a480 comp rx=0 tx=0).stop 2024-09-21T09:41:57.291 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.288+0000 7f45945fd640 1 -- 172.21.15.92:0/2647327299 shutdown_connections 2024-09-21T09:41:57.291 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.288+0000 7f45945fd640 1 --2- 172.21.15.92:0/2647327299 >> v2:172.21.15.92:3300/0 conn(0x7f458c1318a0 0x7f458c133c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.291 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.288+0000 7f45945fd640 1 --2- 172.21.15.92:0/2647327299 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f458c0971e0 0x7f458c131360 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.291 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.288+0000 7f45945fd640 1 -- 172.21.15.92:0/2647327299 >> 172.21.15.92:0/2647327299 conn(0x7f458c09c710 msgr2=0x7f458c09cb20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:57.291 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.288+0000 7f45945fd640 1 -- 172.21.15.92:0/2647327299 shutdown_connections 2024-09-21T09:41:57.291 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.288+0000 7f45945fd640 1 -- 172.21.15.92:0/2647327299 wait complete. 2024-09-21T09:41:57.293 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.289+0000 7f45945fd640 1 Processor -- start 2024-09-21T09:41:57.293 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.289+0000 7f45945fd640 1 -- start start 2024-09-21T09:41:57.293 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.290+0000 7f45945fd640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f458c0966f0 0x7f458c08db30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.293 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.290+0000 7f45945fd640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f458c0971e0 0x7f458c08e070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.293 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.290+0000 7f45945fd640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f458c1318a0 0x7f458c08e5b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:57.293 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.290+0000 7f45945fd640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f458c134a40 con 0x7f458c0971e0 2024-09-21T09:41:57.293 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.290+0000 7f45945fd640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f458c1348c0 con 0x7f458c1318a0 2024-09-21T09:41:57.293 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.290+0000 7f45945fd640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f458c134bc0 con 0x7f458c0966f0 2024-09-21T09:41:57.293 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.290+0000 7f4591b98640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f458c0971e0 0x7f458c08e070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.290+0000 7f4591b98640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f458c0971e0 0x7f458c08e070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:40424/0 (socket says 172.21.15.92:40424) 2024-09-21T09:41:57.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.290+0000 7f4591b98640 1 -- 172.21.15.92:0/4047317400 learned_addr learned my addr 172.21.15.92:0/4047317400 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:57.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.290+0000 7f4592399640 1 --2- 172.21.15.92:0/4047317400 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f458c0966f0 0x7f458c08db30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.290+0000 7f4592b9a640 1 --2- 172.21.15.92:0/4047317400 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f458c1318a0 0x7f458c08e5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:57.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.290+0000 7f4591b98640 1 -- 172.21.15.92:0/4047317400 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f458c0966f0 msgr2=0x7f458c08db30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.290+0000 7f4591b98640 1 --2- 172.21.15.92:0/4047317400 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f458c0966f0 0x7f458c08db30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.290+0000 7f4591b98640 1 -- 172.21.15.92:0/4047317400 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f458c1318a0 msgr2=0x7f458c08e5b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.290+0000 7f4591b98640 1 --2- 172.21.15.92:0/4047317400 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f458c1318a0 0x7f458c08e5b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.290+0000 7f4591b98640 1 -- 172.21.15.92:0/4047317400 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f458c08fe10 con 0x7f458c0971e0 2024-09-21T09:41:57.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.290+0000 7f4591b98640 1 --2- 172.21.15.92:0/4047317400 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f458c0971e0 0x7f458c08e070 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f457c009950 tx=0x7f457c0045e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:57.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.291+0000 7f4582ffd640 1 -- 172.21.15.92:0/4047317400 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f457c005020 con 0x7f458c0971e0 2024-09-21T09:41:57.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.291+0000 7f45945fd640 1 -- 172.21.15.92:0/4047317400 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f458c08f5e0 con 0x7f458c0971e0 2024-09-21T09:41:57.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.291+0000 7f4582ffd640 1 -- 172.21.15.92:0/4047317400 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f457c0052b0 con 0x7f458c0971e0 2024-09-21T09:41:57.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.291+0000 7f4582ffd640 1 -- 172.21.15.92:0/4047317400 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f457c03f7c0 con 0x7f458c0971e0 2024-09-21T09:41:57.295 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.291+0000 7f45945fd640 1 -- 172.21.15.92:0/4047317400 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f458c08ed70 con 0x7f458c0971e0 2024-09-21T09:41:57.295 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.292+0000 7f45945fd640 1 -- 172.21.15.92:0/4047317400 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f458c096ad0 con 0x7f458c0971e0 2024-09-21T09:41:57.295 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.292+0000 7f4582ffd640 1 -- 172.21.15.92:0/4047317400 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f457c039040 con 0x7f458c0971e0 2024-09-21T09:41:57.295 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.292+0000 7f4582ffd640 1 -- 172.21.15.92:0/4047317400 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7f457c082c30 con 0x7f458c0971e0 2024-09-21T09:41:57.298 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.296+0000 7f4582ffd640 1 -- 172.21.15.92:0/4047317400 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f457c081d80 con 0x7f458c0971e0 2024-09-21T09:41:57.414 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.412+0000 7fa6bcef0640 1 -- 172.21.15.92:0/2782163908 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7fa6b80962f0 con 0x7fa6b812ed30 2024-09-21T09:41:57.414 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.412+0000 7fa6a6ffd640 1 -- 172.21.15.92:0/2782163908 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fa6ac03cc70 con 0x7fa6b812ed30 2024-09-21T09:41:57.414 INFO:teuthology.orchestra.run.smithi092.stdout:47244640261 2024-09-21T09:41:57.416 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.414+0000 7fa6bcef0640 1 -- 172.21.15.92:0/2782163908 >> v1:172.21.15.92:6800/1280404317 conn(0x7fa69804c430 legacy=0x7fa69804e8f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.416 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.414+0000 7fa6bcef0640 1 -- 172.21.15.92:0/2782163908 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fa6b812ed30 msgr2=0x7fa6b8129d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.416 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.414+0000 7fa6bcef0640 1 --2- 172.21.15.92:0/2782163908 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fa6b812ed30 0x7fa6b8129d60 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7fa6ac00d740 tx=0x7fa6ac00dc00 comp rx=0 tx=0).stop 2024-09-21T09:41:57.416 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.414+0000 7fa6bcef0640 1 -- 172.21.15.92:0/2782163908 shutdown_connections 2024-09-21T09:41:57.416 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.414+0000 7fa6bcef0640 1 --2- 172.21.15.92:0/2782163908 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fa6b8131660 0x7fa6b812a2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.416 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.414+0000 7fa6bcef0640 1 --2- 172.21.15.92:0/2782163908 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fa6b812ed30 0x7fa6b8129d60 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.416 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.414+0000 7fa6bcef0640 1 --2- 172.21.15.92:0/2782163908 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fa6b8095f10 0x7fa6b8129820 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.416 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.414+0000 7fa6bcef0640 1 -- 172.21.15.92:0/2782163908 >> 172.21.15.92:0/2782163908 conn(0x7fa6b809c590 msgr2=0x7fa6b809a190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:57.416 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.414+0000 7fa6bcef0640 1 -- 172.21.15.92:0/2782163908 shutdown_connections 2024-09-21T09:41:57.416 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.414+0000 7fa6bcef0640 1 -- 172.21.15.92:0/2782163908 wait complete. 2024-09-21T09:41:57.427 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640262 got 47244640261 for osd.0 2024-09-21T09:41:57.431 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.430+0000 7fdc5ce24640 1 -- 172.21.15.92:0/1693909738 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7fdc3c005580 con 0x7fdc580963e0 2024-09-21T09:41:57.431 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.430+0000 7fdc46ffd640 1 -- 172.21.15.92:0/1693909738 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fdc4c03ca10 con 0x7fdc580963e0 2024-09-21T09:41:57.432 INFO:teuthology.orchestra.run.smithi092.stdout:38654705667 2024-09-21T09:41:57.433 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.432+0000 7fdc5ce24640 1 -- 172.21.15.92:0/1693909738 >> v1:172.21.15.92:6800/1280404317 conn(0x7fdc38047fd0 legacy=0x7fdc3804a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.434 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.432+0000 7fdc5ce24640 1 -- 172.21.15.92:0/1693909738 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fdc580963e0 msgr2=0x7fdc58122690 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.434 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.432+0000 7fdc5ce24640 1 --2- 172.21.15.92:0/1693909738 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fdc580963e0 0x7fdc58122690 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7fdc4c00c450 tx=0x7fdc4c00c910 comp rx=0 tx=0).stop 2024-09-21T09:41:57.434 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.432+0000 7fdc5ce24640 1 -- 172.21.15.92:0/1693909738 shutdown_connections 2024-09-21T09:41:57.434 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.432+0000 7fdc5ce24640 1 --2- 172.21.15.92:0/1693909738 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fdc5812b230 0x7fdc58122bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.434 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.432+0000 7fdc5ce24640 1 --2- 172.21.15.92:0/1693909738 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fdc580963e0 0x7fdc58122690 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.434 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.432+0000 7fdc5ce24640 1 --2- 172.21.15.92:0/1693909738 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fdc58070b10 0x7fdc58122150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.434 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.432+0000 7fdc5ce24640 1 -- 172.21.15.92:0/1693909738 >> 172.21.15.92:0/1693909738 conn(0x7fdc5809cd80 msgr2=0x7fdc58097bf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:57.434 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.432+0000 7fdc5ce24640 1 -- 172.21.15.92:0/1693909738 shutdown_connections 2024-09-21T09:41:57.434 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.432+0000 7fdc5ce24640 1 -- 172.21.15.92:0/1693909738 wait complete. 2024-09-21T09:41:57.440 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.439+0000 7fc98038a640 1 -- 172.21.15.92:0/3961855085 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7fc97812b340 con 0x7fc97812af60 2024-09-21T09:41:57.441 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.439+0000 7fc96effd640 1 -- 172.21.15.92:0/3961855085 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fc96803d6c0 con 0x7fc97812af60 2024-09-21T09:41:57.441 INFO:teuthology.orchestra.run.smithi092.stdout:47244640261 2024-09-21T09:41:57.443 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.441+0000 7fc98038a640 1 -- 172.21.15.92:0/3961855085 >> v1:172.21.15.92:6800/1280404317 conn(0x7fc95c04c430 legacy=0x7fc95c04e8f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.443 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.441+0000 7fc98038a640 1 -- 172.21.15.92:0/3961855085 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc97812af60 msgr2=0x7fc978122400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.443 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.441+0000 7fc98038a640 1 --2- 172.21.15.92:0/3961855085 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc97812af60 0x7fc978122400 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7fc96800d950 tx=0x7fc96800de10 comp rx=0 tx=0).stop 2024-09-21T09:41:57.443 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.442+0000 7fc98038a640 1 -- 172.21.15.92:0/3961855085 shutdown_connections 2024-09-21T09:41:57.444 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.442+0000 7fc98038a640 1 --2- 172.21.15.92:0/3961855085 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc97812b880 0x7fc978122940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.444 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.442+0000 7fc98038a640 1 --2- 172.21.15.92:0/3961855085 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc97812af60 0x7fc978122400 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.444 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.442+0000 7fc98038a640 1 --2- 172.21.15.92:0/3961855085 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc9780965f0 0x7fc978121ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.444 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.442+0000 7fc98038a640 1 -- 172.21.15.92:0/3961855085 >> 172.21.15.92:0/3961855085 conn(0x7fc97809ac70 msgr2=0x7fc97812c470 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:57.444 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.442+0000 7fc98038a640 1 -- 172.21.15.92:0/3961855085 shutdown_connections 2024-09-21T09:41:57.444 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.442+0000 7fc98038a640 1 -- 172.21.15.92:0/3961855085 wait complete. 2024-09-21T09:41:57.445 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705669 got 38654705667 for osd.3 2024-09-21T09:41:57.447 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.446+0000 7f45945fd640 1 -- 172.21.15.92:0/4047317400 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7f458c09dca0 con 0x7f458c0971e0 2024-09-21T09:41:57.448 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.446+0000 7f4582ffd640 1 -- 172.21.15.92:0/4047317400 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f457c0818c0 con 0x7f458c0971e0 2024-09-21T09:41:57.449 INFO:teuthology.orchestra.run.smithi092.stdout:47244640259 2024-09-21T09:41:57.450 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.448+0000 7f45945fd640 1 -- 172.21.15.92:0/4047317400 >> v1:172.21.15.92:6800/1280404317 conn(0x7f4570047fd0 legacy=0x7f457004a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.450 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.448+0000 7f45945fd640 1 -- 172.21.15.92:0/4047317400 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f458c0971e0 msgr2=0x7f458c08e070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:57.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.448+0000 7f45945fd640 1 --2- 172.21.15.92:0/4047317400 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f458c0971e0 0x7f458c08e070 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f457c009950 tx=0x7f457c0045e0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.448+0000 7f45945fd640 1 -- 172.21.15.92:0/4047317400 shutdown_connections 2024-09-21T09:41:57.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.448+0000 7f45945fd640 1 --2- 172.21.15.92:0/4047317400 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f458c1318a0 0x7f458c08e5b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.448+0000 7f45945fd640 1 --2- 172.21.15.92:0/4047317400 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f458c0971e0 0x7f458c08e070 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.448+0000 7f45945fd640 1 --2- 172.21.15.92:0/4047317400 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f458c0966f0 0x7f458c08db30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:57.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.448+0000 7f45945fd640 1 -- 172.21.15.92:0/4047317400 >> 172.21.15.92:0/4047317400 conn(0x7f458c09c710 msgr2=0x7f458c06f8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:57.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.448+0000 7f45945fd640 1 -- 172.21.15.92:0/4047317400 shutdown_connections 2024-09-21T09:41:57.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:57.448+0000 7f45945fd640 1 -- 172.21.15.92:0/4047317400 wait complete. 2024-09-21T09:41:57.454 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640262 got 47244640261 for osd.2 2024-09-21T09:41:57.460 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640260 got 47244640259 for osd.1 2024-09-21T09:41:58.428 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T09:41:58.446 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T09:41:58.455 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T09:41:58.461 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T09:41:58.539 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.536+0000 7fce0230f640 1 -- 172.21.15.92:0/2305656866 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcdfc096830 msgr2=0x7fcdfc12ba20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:58.539 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.536+0000 7fce0230f640 1 --2- 172.21.15.92:0/2305656866 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcdfc096830 0x7fcdfc12ba20 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7fcdf0009830 tx=0x7fcdf002a3f0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.539 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.537+0000 7fce0230f640 1 -- 172.21.15.92:0/2305656866 shutdown_connections 2024-09-21T09:41:58.539 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.537+0000 7fce0230f640 1 --2- 172.21.15.92:0/2305656866 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcdfc096830 0x7fcdfc12ba20 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.539 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.537+0000 7fce0230f640 1 --2- 172.21.15.92:0/2305656866 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcdfc095f10 0x7fcdfc0962f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.539 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.537+0000 7fce0230f640 1 -- 172.21.15.92:0/2305656866 >> 172.21.15.92:0/2305656866 conn(0x7fcdfc09c590 msgr2=0x7fcdfc09c9a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:58.539 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.537+0000 7fce0230f640 1 -- 172.21.15.92:0/2305656866 shutdown_connections 2024-09-21T09:41:58.539 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.537+0000 7fce0230f640 1 -- 172.21.15.92:0/2305656866 wait complete. 2024-09-21T09:41:58.540 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.538+0000 7fce0230f640 1 Processor -- start 2024-09-21T09:41:58.540 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.538+0000 7fce0230f640 1 -- start start 2024-09-21T09:41:58.541 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.539+0000 7fce0230f640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcdfc095f10 0x7fcdfc1c1c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:58.541 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.539+0000 7fce0230f640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcdfc096830 0x7fcdfc1c2170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:58.541 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.539+0000 7fce0230f640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcdfc12bf60 0x7fcdfc1c26b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:58.541 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.539+0000 7fcdfb7fe640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcdfc096830 0x7fcdfc1c2170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:58.542 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.539+0000 7fce0230f640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fcdfc1347e0 con 0x7fcdfc096830 2024-09-21T09:41:58.542 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.539+0000 7fce0230f640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fcdfc134660 con 0x7fcdfc12bf60 2024-09-21T09:41:58.542 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.539+0000 7fce0230f640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fcdfc134960 con 0x7fcdfc095f10 2024-09-21T09:41:58.542 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.539+0000 7fcdfb7fe640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcdfc096830 0x7fcdfc1c2170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:40434/0 (socket says 172.21.15.92:40434) 2024-09-21T09:41:58.542 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.539+0000 7fcdfb7fe640 1 -- 172.21.15.92:0/1227262468 learned_addr learned my addr 172.21.15.92:0/1227262468 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:58.542 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.539+0000 7fcdfbfff640 1 --2- 172.21.15.92:0/1227262468 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcdfc095f10 0x7fcdfc1c1c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:58.542 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.539+0000 7fce008ac640 1 --2- 172.21.15.92:0/1227262468 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcdfc12bf60 0x7fcdfc1c26b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:58.542 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.539+0000 7fcdfb7fe640 1 -- 172.21.15.92:0/1227262468 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcdfc095f10 msgr2=0x7fcdfc1c1c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:58.542 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.539+0000 7fcdfb7fe640 1 --2- 172.21.15.92:0/1227262468 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcdfc095f10 0x7fcdfc1c1c30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.542 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.539+0000 7fcdfb7fe640 1 -- 172.21.15.92:0/1227262468 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcdfc12bf60 msgr2=0x7fcdfc1c26b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:58.542 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.539+0000 7fcdfb7fe640 1 --2- 172.21.15.92:0/1227262468 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcdfc12bf60 0x7fcdfc1c26b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.542 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.539+0000 7fcdfb7fe640 1 -- 172.21.15.92:0/1227262468 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcdfc0719a0 con 0x7fcdfc096830 2024-09-21T09:41:58.542 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.539+0000 7fcdfb7fe640 1 --2- 172.21.15.92:0/1227262468 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcdfc096830 0x7fcdfc1c2170 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7fcdf0009950 tx=0x7fcdf0032b90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:58.543 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.539+0000 7fcdf8ff9640 1 -- 172.21.15.92:0/1227262468 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcdf0005c90 con 0x7fcdfc096830 2024-09-21T09:41:58.543 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.540+0000 7fcdf8ff9640 1 -- 172.21.15.92:0/1227262468 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fcdf0047070 con 0x7fcdfc096830 2024-09-21T09:41:58.543 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.540+0000 7fce0230f640 1 -- 172.21.15.92:0/1227262468 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fcdfc071560 con 0x7fcdfc096830 2024-09-21T09:41:58.543 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.540+0000 7fcdf8ff9640 1 -- 172.21.15.92:0/1227262468 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcdf0003850 con 0x7fcdfc096830 2024-09-21T09:41:58.543 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.540+0000 7fce0230f640 1 -- 172.21.15.92:0/1227262468 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fcdfc06f960 con 0x7fcdfc096830 2024-09-21T09:41:58.543 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.541+0000 7fce0230f640 1 -- 172.21.15.92:0/1227262468 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fcdfc0962f0 con 0x7fcdfc096830 2024-09-21T09:41:58.543 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.541+0000 7fcdf8ff9640 1 -- 172.21.15.92:0/1227262468 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fcdf0039040 con 0x7fcdfc096830 2024-09-21T09:41:58.544 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.541+0000 7fcdf8ff9640 1 -- 172.21.15.92:0/1227262468 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7fcdf00870d0 con 0x7fcdfc096830 2024-09-21T09:41:58.550 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.548+0000 7fcdf8ff9640 1 -- 172.21.15.92:0/1227262468 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7fcdf0086220 con 0x7fcdfc096830 2024-09-21T09:41:58.580 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.578+0000 7f97380d7640 1 -- 172.21.15.92:0/1276511689 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f973008d270 msgr2=0x7f9730132fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:58.580 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.578+0000 7f97380d7640 1 --2- 172.21.15.92:0/1276511689 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f973008d270 0x7f9730132fb0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f9720009900 tx=0x7f972002a4f0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.580 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.578+0000 7f97380d7640 1 -- 172.21.15.92:0/1276511689 shutdown_connections 2024-09-21T09:41:58.580 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.578+0000 7f97380d7640 1 --2- 172.21.15.92:0/1276511689 >> v2:172.21.15.92:3300/0 conn(0x7f97301334f0 0x7f97301358e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.580 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.578+0000 7f97380d7640 1 --2- 172.21.15.92:0/1276511689 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f973008d270 0x7f9730132fb0 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.580 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.578+0000 7f97380d7640 1 -- 172.21.15.92:0/1276511689 >> 172.21.15.92:0/1276511689 conn(0x7f97300a0c80 msgr2=0x7f973008b000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:58.581 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.578+0000 7f97380d7640 1 -- 172.21.15.92:0/1276511689 shutdown_connections 2024-09-21T09:41:58.581 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.578+0000 7f97380d7640 1 -- 172.21.15.92:0/1276511689 wait complete. 2024-09-21T09:41:58.581 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.579+0000 7f97380d7640 1 Processor -- start 2024-09-21T09:41:58.581 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.579+0000 7f97380d7640 1 -- start start 2024-09-21T09:41:58.582 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.580+0000 7fe8b7353640 1 -- 172.21.15.92:0/1444385882 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe8b0131a70 msgr2=0x7fe8b0133e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:58.582 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.580+0000 7fe8b7353640 1 --2- 172.21.15.92:0/1444385882 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe8b0131a70 0x7fe8b0133e60 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7fe8a8009be0 tx=0x7fe8a802a800 comp rx=0 tx=0).stop 2024-09-21T09:41:58.582 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.580+0000 7fe8b7353640 1 -- 172.21.15.92:0/1444385882 shutdown_connections 2024-09-21T09:41:58.582 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.580+0000 7fe8b7353640 1 --2- 172.21.15.92:0/1444385882 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe8b0131a70 0x7fe8b0133e60 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.582 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.580+0000 7fe8b7353640 1 --2- 172.21.15.92:0/1444385882 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe8b00966f0 0x7fe8b0096ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.582 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.580+0000 7fe8b7353640 1 -- 172.21.15.92:0/1444385882 >> 172.21.15.92:0/1444385882 conn(0x7fe8b009c710 msgr2=0x7fe8b009cb20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:58.582 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.580+0000 7fe8b7353640 1 -- 172.21.15.92:0/1444385882 shutdown_connections 2024-09-21T09:41:58.583 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.580+0000 7fe8b7353640 1 -- 172.21.15.92:0/1444385882 wait complete. 2024-09-21T09:41:58.584 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.580+0000 7f97380d7640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f973008c780 0x7f973008eb60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:58.584 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.580+0000 7f97380d7640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f973008d270 0x7f973008f0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:58.584 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.580+0000 7f9735e73640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f973008c780 0x7f973008eb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:58.584 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.580+0000 7f97380d7640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f97301334f0 0x7f973008f5e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:58.584 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.580+0000 7f9735e73640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f973008c780 0x7f973008eb60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:40452/0 (socket says 172.21.15.92:40452) 2024-09-21T09:41:58.584 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.580+0000 7f9735672640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f973008d270 0x7f973008f0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:58.584 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.580+0000 7f9735e73640 1 -- 172.21.15.92:0/1162562634 learned_addr learned my addr 172.21.15.92:0/1162562634 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:58.584 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.580+0000 7f97380d7640 1 -- 172.21.15.92:0/1162562634 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f97301368f0 con 0x7f973008c780 2024-09-21T09:41:58.584 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.580+0000 7f97380d7640 1 -- 172.21.15.92:0/1162562634 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f9730136770 con 0x7f97301334f0 2024-09-21T09:41:58.584 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.580+0000 7f97380d7640 1 -- 172.21.15.92:0/1162562634 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f9730136a70 con 0x7f973008d270 2024-09-21T09:41:58.584 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.580+0000 7f9736674640 1 --2- 172.21.15.92:0/1162562634 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f97301334f0 0x7f973008f5e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:58.584 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.581+0000 7f9735e73640 1 -- 172.21.15.92:0/1162562634 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f973008d270 msgr2=0x7f973008f0a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:58.584 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.581+0000 7f9735e73640 1 --2- 172.21.15.92:0/1162562634 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f973008d270 0x7f973008f0a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.585 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.581+0000 7f9735e73640 1 -- 172.21.15.92:0/1162562634 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f97301334f0 msgr2=0x7f973008f5e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:58.585 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.581+0000 7f9735e73640 1 --2- 172.21.15.92:0/1162562634 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f97301334f0 0x7f973008f5e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.585 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.581+0000 7f9735e73640 1 -- 172.21.15.92:0/1162562634 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9730092790 con 0x7f973008c780 2024-09-21T09:41:58.585 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.581+0000 7f9735672640 1 --2- 172.21.15.92:0/1162562634 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f973008d270 0x7f973008f0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:58.585 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.581+0000 7f9736674640 1 --2- 172.21.15.92:0/1162562634 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f97301334f0 0x7f973008f5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:58.585 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.581+0000 7fe8b7353640 1 Processor -- start 2024-09-21T09:41:58.585 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.581+0000 7fe8b7353640 1 -- start start 2024-09-21T09:41:58.586 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.581+0000 7f9735e73640 1 --2- 172.21.15.92:0/1162562634 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f973008c780 0x7f973008eb60 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7f972c00c3c0 tx=0x7f972c00c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:58.586 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.581+0000 7f97267fc640 1 -- 172.21.15.92:0/1162562634 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f972c017070 con 0x7f973008c780 2024-09-21T09:41:58.586 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.581+0000 7f97267fc640 1 -- 172.21.15.92:0/1162562634 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f972c0035f0 con 0x7f973008c780 2024-09-21T09:41:58.586 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.581+0000 7f97267fc640 1 -- 172.21.15.92:0/1162562634 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f972c003a40 con 0x7f973008c780 2024-09-21T09:41:58.586 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.582+0000 7f97380d7640 1 -- 172.21.15.92:0/1162562634 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9730091010 con 0x7f973008c780 2024-09-21T09:41:58.587 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.582+0000 7fe8b7353640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe8b00966f0 0x7fe8b008e280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:58.587 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.582+0000 7f97380d7640 1 -- 172.21.15.92:0/1162562634 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f97301c9f30 con 0x7f973008c780 2024-09-21T09:41:58.588 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.582+0000 7fe8b50ef640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe8b00966f0 0x7fe8b008e280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:58.588 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.582+0000 7fe8b50ef640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe8b00966f0 0x7fe8b008e280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:46950/0 (socket says 172.21.15.92:46950) 2024-09-21T09:41:58.588 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.582+0000 7fe8b50ef640 1 -- 172.21.15.92:0/1399167458 learned_addr learned my addr 172.21.15.92:0/1399167458 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:58.588 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.582+0000 7fe8b7353640 1 --2- 172.21.15.92:0/1399167458 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe8b0097010 0x7fe8b008e7c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:58.589 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.582+0000 7fe8b7353640 1 --2- 172.21.15.92:0/1399167458 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe8b0131a70 0x7fe8b008ed00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:58.589 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.582+0000 7fe8b7353640 1 -- 172.21.15.92:0/1399167458 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fe8b0134c00 con 0x7fe8b0097010 2024-09-21T09:41:58.589 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.582+0000 7fe8b48ee640 1 --2- 172.21.15.92:0/1399167458 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe8b0097010 0x7fe8b008e7c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:58.589 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.582+0000 7fe8b58f0640 1 --2- 172.21.15.92:0/1399167458 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe8b0131a70 0x7fe8b008ed00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:58.589 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.582+0000 7fe8b7353640 1 -- 172.21.15.92:0/1399167458 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fe8b0134a80 con 0x7fe8b0131a70 2024-09-21T09:41:58.589 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.582+0000 7fe8b7353640 1 -- 172.21.15.92:0/1399167458 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fe8b0134d80 con 0x7fe8b00966f0 2024-09-21T09:41:58.589 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.583+0000 7fe8b50ef640 1 -- 172.21.15.92:0/1399167458 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe8b0131a70 msgr2=0x7fe8b008ed00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:58.589 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.583+0000 7fe8b50ef640 1 --2- 172.21.15.92:0/1399167458 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe8b0131a70 0x7fe8b008ed00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.589 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.583+0000 7f97380d7640 1 -- 172.21.15.92:0/1162562634 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9730091bc0 con 0x7f973008c780 2024-09-21T09:41:58.589 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.583+0000 7f97267fc640 1 -- 172.21.15.92:0/1162562634 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f972c01a070 con 0x7f973008c780 2024-09-21T09:41:58.590 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.583+0000 7fe8b50ef640 1 -- 172.21.15.92:0/1399167458 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe8b0097010 msgr2=0x7fe8b008e7c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:58.591 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.583+0000 7fe8b50ef640 1 --2- 172.21.15.92:0/1399167458 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe8b0097010 0x7fe8b008e7c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.591 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.583+0000 7fe8b50ef640 1 -- 172.21.15.92:0/1399167458 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe8b00902f0 con 0x7fe8b00966f0 2024-09-21T09:41:58.591 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.583+0000 7fe8b48ee640 1 --2- 172.21.15.92:0/1399167458 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe8b0097010 0x7fe8b008e7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:41:58.591 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.583+0000 7fe8b50ef640 1 --2- 172.21.15.92:0/1399167458 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe8b00966f0 0x7fe8b008e280 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7fe8ac00d950 tx=0x7fe8ac00de10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:58.591 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.583+0000 7fe8a5ffb640 1 -- 172.21.15.92:0/1399167458 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe8ac017030 con 0x7fe8b00966f0 2024-09-21T09:41:58.591 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.583+0000 7fe8b7353640 1 -- 172.21.15.92:0/1399167458 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe8b008fac0 con 0x7fe8b00966f0 2024-09-21T09:41:58.591 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.583+0000 7f97267fc640 1 -- 172.21.15.92:0/1162562634 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7f972c05ccf0 con 0x7f973008c780 2024-09-21T09:41:58.592 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.583+0000 7fe8a5ffb640 1 -- 172.21.15.92:0/1399167458 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fe8ac009b20 con 0x7fe8b00966f0 2024-09-21T09:41:58.592 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.583+0000 7fe8a5ffb640 1 -- 172.21.15.92:0/1399167458 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe8ac004020 con 0x7fe8b00966f0 2024-09-21T09:41:58.592 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.584+0000 7fe8b7353640 1 -- 172.21.15.92:0/1399167458 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fe8b0090d80 con 0x7fe8b00966f0 2024-09-21T09:41:58.592 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.585+0000 7fe8a5ffb640 1 -- 172.21.15.92:0/1399167458 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fe8ac0041c0 con 0x7fe8b00966f0 2024-09-21T09:41:58.592 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.585+0000 7fe8a5ffb640 1 -- 172.21.15.92:0/1399167458 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7fe8ac062d10 con 0x7fe8b00966f0 2024-09-21T09:41:58.592 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.586+0000 7fe8b7353640 1 -- 172.21.15.92:0/1399167458 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe8b006ccc0 con 0x7fe8b00966f0 2024-09-21T09:41:58.593 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.587+0000 7f97267fc640 1 -- 172.21.15.92:0/1162562634 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f972c03bee0 con 0x7f973008c780 2024-09-21T09:41:58.593 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.589+0000 7fe8a5ffb640 1 -- 172.21.15.92:0/1399167458 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7fe8ac03c420 con 0x7fe8b00966f0 2024-09-21T09:41:58.594 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.593+0000 7f794dffb640 1 -- 172.21.15.92:0/3180038803 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7948032d10 con 0x7f795012b0c0 2024-09-21T09:41:58.595 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.593+0000 7f7955553640 1 -- 172.21.15.92:0/3180038803 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f795012b0c0 msgr2=0x7f795012b530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:58.595 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.593+0000 7f7955553640 1 --2- 172.21.15.92:0/3180038803 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f795012b0c0 0x7f795012b530 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f7948005fd0 tx=0x7f794802a780 comp rx=0 tx=0).stop 2024-09-21T09:41:58.595 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.593+0000 7f7955553640 1 -- 172.21.15.92:0/3180038803 shutdown_connections 2024-09-21T09:41:58.595 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.593+0000 7f7955553640 1 --2- 172.21.15.92:0/3180038803 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f795012b0c0 0x7f795012b530 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.595 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.593+0000 7f7955553640 1 --2- 172.21.15.92:0/3180038803 >> v2:172.21.15.92:3300/0 conn(0x7f7950094f10 0x7f79500952f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.595 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.593+0000 7f7955553640 1 -- 172.21.15.92:0/3180038803 >> 172.21.15.92:0/3180038803 conn(0x7f795009b590 msgr2=0x7f795009b9a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:58.595 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.593+0000 7f7955553640 1 -- 172.21.15.92:0/3180038803 shutdown_connections 2024-09-21T09:41:58.596 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.594+0000 7f7955553640 1 -- 172.21.15.92:0/3180038803 wait complete. 2024-09-21T09:41:58.596 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.595+0000 7f7955553640 1 Processor -- start 2024-09-21T09:41:58.596 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.595+0000 7f7955553640 1 -- start start 2024-09-21T09:41:58.597 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.595+0000 7f7955553640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7950094f10 0x7f795006fe10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:58.597 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.595+0000 7f7955553640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7950095830 0x7f7950070350 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:58.597 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.595+0000 7f7955553640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f795012b0c0 0x7f7950070890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:58.597 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.595+0000 7f794f7fe640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f795012b0c0 0x7f7950070890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:58.597 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.595+0000 7f794f7fe640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f795012b0c0 0x7f7950070890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:41076/0 (socket says 172.21.15.92:41076) 2024-09-21T09:41:58.597 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.595+0000 7f794f7fe640 1 -- 172.21.15.92:0/95102489 learned_addr learned my addr 172.21.15.92:0/95102489 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:58.597 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.595+0000 7f794effd640 1 --2- 172.21.15.92:0/95102489 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7950094f10 0x7f795006fe10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:58.597 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.595+0000 7f7955553640 1 -- 172.21.15.92:0/95102489 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f79501336a0 con 0x7f7950095830 2024-09-21T09:41:58.598 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.595+0000 7f7955553640 1 -- 172.21.15.92:0/95102489 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f7950133520 con 0x7f795012b0c0 2024-09-21T09:41:58.598 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.595+0000 7f7955553640 1 -- 172.21.15.92:0/95102489 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f7950133820 con 0x7f7950094f10 2024-09-21T09:41:58.598 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.595+0000 7f794e7fc640 1 --2- 172.21.15.92:0/95102489 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7950095830 0x7f7950070350 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:58.599 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.596+0000 7f794f7fe640 1 -- 172.21.15.92:0/95102489 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7950094f10 msgr2=0x7f795006fe10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:58.599 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.596+0000 7f794f7fe640 1 --2- 172.21.15.92:0/95102489 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7950094f10 0x7f795006fe10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.599 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.596+0000 7f794f7fe640 1 -- 172.21.15.92:0/95102489 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7950095830 msgr2=0x7f7950070350 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:58.599 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.596+0000 7f794f7fe640 1 --2- 172.21.15.92:0/95102489 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7950095830 0x7f7950070350 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.599 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.596+0000 7f794f7fe640 1 -- 172.21.15.92:0/95102489 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7950071980 con 0x7f795012b0c0 2024-09-21T09:41:58.599 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.596+0000 7f794e7fc640 1 --2- 172.21.15.92:0/95102489 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7950095830 0x7f7950070350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:58.599 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.596+0000 7f794effd640 1 --2- 172.21.15.92:0/95102489 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7950094f10 0x7f795006fe10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T09:41:58.599 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.596+0000 7f794f7fe640 1 --2- 172.21.15.92:0/95102489 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f795012b0c0 0x7f7950070890 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f7948005fd0 tx=0x7f7948003f90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:58.599 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.596+0000 7f79377fe640 1 -- 172.21.15.92:0/95102489 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7948040070 con 0x7f795012b0c0 2024-09-21T09:41:58.599 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.596+0000 7f7955553640 1 -- 172.21.15.92:0/95102489 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7950132020 con 0x7f795012b0c0 2024-09-21T09:41:58.600 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.596+0000 7f79377fe640 1 -- 172.21.15.92:0/95102489 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f7948004680 con 0x7f795012b0c0 2024-09-21T09:41:58.600 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.597+0000 7f79377fe640 1 -- 172.21.15.92:0/95102489 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f794804a060 con 0x7f795012b0c0 2024-09-21T09:41:58.600 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.597+0000 7f7955553640 1 -- 172.21.15.92:0/95102489 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f7950072d40 con 0x7f795012b0c0 2024-09-21T09:41:58.600 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.598+0000 7f79377fe640 1 -- 172.21.15.92:0/95102489 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f7948043070 con 0x7f795012b0c0 2024-09-21T09:41:58.600 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.598+0000 7f79377fe640 1 -- 172.21.15.92:0/95102489 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(18..18 src has 1..18) ==== 2868+0+0 (secure 0 0 0) 0x7f794807ee40 con 0x7f795012b0c0 2024-09-21T09:41:58.600 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.598+0000 7f7955553640 1 -- 172.21.15.92:0/95102489 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f79500731d0 con 0x7f795012b0c0 2024-09-21T09:41:58.607 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.606+0000 7f79377fe640 1 -- 172.21.15.92:0/95102489 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f79500731d0 con 0x7f795012b0c0 2024-09-21T09:41:58.722 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.720+0000 7fce0230f640 1 -- 172.21.15.92:0/1227262468 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7fcdfc0710c0 con 0x7fcdfc096830 2024-09-21T09:41:58.722 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.720+0000 7fcdf8ff9640 1 -- 172.21.15.92:0/1227262468 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fcdf0085d60 con 0x7fcdfc096830 2024-09-21T09:41:58.722 INFO:teuthology.orchestra.run.smithi092.stdout:47244640261 2024-09-21T09:41:58.724 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.722+0000 7fce0230f640 1 -- 172.21.15.92:0/1227262468 >> v1:172.21.15.92:6800/1280404317 conn(0x7fcde8047fd0 legacy=0x7fcde804a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:58.724 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.722+0000 7fce0230f640 1 -- 172.21.15.92:0/1227262468 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcdfc096830 msgr2=0x7fcdfc1c2170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:58.724 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.722+0000 7fce0230f640 1 --2- 172.21.15.92:0/1227262468 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcdfc096830 0x7fcdfc1c2170 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7fcdf0009950 tx=0x7fcdf0032b90 comp rx=0 tx=0).stop 2024-09-21T09:41:58.724 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.722+0000 7fce0230f640 1 -- 172.21.15.92:0/1227262468 shutdown_connections 2024-09-21T09:41:58.724 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.722+0000 7fce0230f640 1 --2- 172.21.15.92:0/1227262468 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcdfc12bf60 0x7fcdfc1c26b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.724 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.722+0000 7fce0230f640 1 --2- 172.21.15.92:0/1227262468 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcdfc096830 0x7fcdfc1c2170 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.724 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.722+0000 7fce0230f640 1 --2- 172.21.15.92:0/1227262468 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcdfc095f10 0x7fcdfc1c1c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.724 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.722+0000 7fce0230f640 1 -- 172.21.15.92:0/1227262468 >> 172.21.15.92:0/1227262468 conn(0x7fcdfc09c590 msgr2=0x7fcdfc125340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:58.724 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.722+0000 7fce0230f640 1 -- 172.21.15.92:0/1227262468 shutdown_connections 2024-09-21T09:41:58.725 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.722+0000 7fce0230f640 1 -- 172.21.15.92:0/1227262468 wait complete. 2024-09-21T09:41:58.738 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640262 got 47244640261 for osd.0 2024-09-21T09:41:58.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.738+0000 7f97380d7640 1 -- 172.21.15.92:0/1162562634 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7f9730126e20 con 0x7f973008c780 2024-09-21T09:41:58.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.739+0000 7f97267fc640 1 -- 172.21.15.92:0/1162562634 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f972c03c140 con 0x7f973008c780 2024-09-21T09:41:58.741 INFO:teuthology.orchestra.run.smithi092.stdout:38654705667 2024-09-21T09:41:58.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.741+0000 7f97380d7640 1 -- 172.21.15.92:0/1162562634 >> v1:172.21.15.92:6800/1280404317 conn(0x7f9714047fd0 legacy=0x7f971404a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:58.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.741+0000 7f97380d7640 1 -- 172.21.15.92:0/1162562634 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f973008c780 msgr2=0x7f973008eb60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:58.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.741+0000 7f97380d7640 1 --2- 172.21.15.92:0/1162562634 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f973008c780 0x7f973008eb60 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7f972c00c3c0 tx=0x7f972c00c880 comp rx=0 tx=0).stop 2024-09-21T09:41:58.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.741+0000 7f97380d7640 1 -- 172.21.15.92:0/1162562634 shutdown_connections 2024-09-21T09:41:58.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.741+0000 7f97380d7640 1 --2- 172.21.15.92:0/1162562634 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f97301334f0 0x7f973008f5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.741+0000 7f97380d7640 1 --2- 172.21.15.92:0/1162562634 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f973008d270 0x7f973008f0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.741+0000 7f97380d7640 1 --2- 172.21.15.92:0/1162562634 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f973008c780 0x7f973008eb60 unknown :-1 s=CLOSED pgs=111 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.741+0000 7f97380d7640 1 -- 172.21.15.92:0/1162562634 >> 172.21.15.92:0/1162562634 conn(0x7f97300a0c80 msgr2=0x7f973012b010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:58.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.741+0000 7f97380d7640 1 -- 172.21.15.92:0/1162562634 shutdown_connections 2024-09-21T09:41:58.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.741+0000 7f97380d7640 1 -- 172.21.15.92:0/1162562634 wait complete. 2024-09-21T09:41:58.751 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.749+0000 7fe8b7353640 1 -- 172.21.15.92:0/1399167458 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7fe8b0094180 con 0x7fe8b00966f0 2024-09-21T09:41:58.751 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.750+0000 7fe8a5ffb640 1 -- 172.21.15.92:0/1399167458 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fe8ac05c220 con 0x7fe8b00966f0 2024-09-21T09:41:58.752 INFO:teuthology.orchestra.run.smithi092.stdout:47244640259 2024-09-21T09:41:58.753 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705669 got 38654705667 for osd.3 2024-09-21T09:41:58.754 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.752+0000 7fe8b7353640 1 -- 172.21.15.92:0/1399167458 >> v1:172.21.15.92:6800/1280404317 conn(0x7fe894047fd0 legacy=0x7fe89404a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:58.754 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.752+0000 7fe8b7353640 1 -- 172.21.15.92:0/1399167458 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe8b00966f0 msgr2=0x7fe8b008e280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:58.754 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.752+0000 7fe8b7353640 1 --2- 172.21.15.92:0/1399167458 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe8b00966f0 0x7fe8b008e280 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7fe8ac00d950 tx=0x7fe8ac00de10 comp rx=0 tx=0).stop 2024-09-21T09:41:58.754 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.752+0000 7fe8b7353640 1 -- 172.21.15.92:0/1399167458 shutdown_connections 2024-09-21T09:41:58.754 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.752+0000 7fe8b7353640 1 --2- 172.21.15.92:0/1399167458 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe8b0131a70 0x7fe8b008ed00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.754 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.752+0000 7fe8b7353640 1 --2- 172.21.15.92:0/1399167458 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe8b0097010 0x7fe8b008e7c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.754 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.752+0000 7fe8b7353640 1 --2- 172.21.15.92:0/1399167458 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe8b00966f0 0x7fe8b008e280 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.755 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.752+0000 7fe8b7353640 1 -- 172.21.15.92:0/1399167458 >> 172.21.15.92:0/1399167458 conn(0x7fe8b009c710 msgr2=0x7fe8b006f8c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:58.755 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.752+0000 7fe8b7353640 1 -- 172.21.15.92:0/1399167458 shutdown_connections 2024-09-21T09:41:58.755 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.752+0000 7fe8b7353640 1 -- 172.21.15.92:0/1399167458 wait complete. 2024-09-21T09:41:58.758 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.756+0000 7f7955553640 1 -- 172.21.15.92:0/95102489 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7f79500736f0 con 0x7f795012b0c0 2024-09-21T09:41:58.758 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.757+0000 7f79377fe640 1 -- 172.21.15.92:0/95102489 <== mon.1 v2:172.21.15.92:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f79500736f0 con 0x7f795012b0c0 2024-09-21T09:41:58.758 INFO:teuthology.orchestra.run.smithi092.stdout:47244640261 2024-09-21T09:41:58.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.758+0000 7f7955553640 1 -- 172.21.15.92:0/95102489 >> v1:172.21.15.92:6800/1280404317 conn(0x7f793c048000 legacy=0x7f793c04a4c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:58.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.758+0000 7f7955553640 1 -- 172.21.15.92:0/95102489 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f795012b0c0 msgr2=0x7f7950070890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:58.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.759+0000 7f7955553640 1 --2- 172.21.15.92:0/95102489 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f795012b0c0 0x7f7950070890 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f7948005fd0 tx=0x7f7948003f90 comp rx=0 tx=0).stop 2024-09-21T09:41:58.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.759+0000 7f7955553640 1 -- 172.21.15.92:0/95102489 shutdown_connections 2024-09-21T09:41:58.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.759+0000 7f7955553640 1 --2- 172.21.15.92:0/95102489 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f795012b0c0 0x7f7950070890 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.759+0000 7f7955553640 1 --2- 172.21.15.92:0/95102489 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7950095830 0x7f7950070350 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.759+0000 7f7955553640 1 --2- 172.21.15.92:0/95102489 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7950094f10 0x7f795006fe10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:58.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.759+0000 7f7955553640 1 -- 172.21.15.92:0/95102489 >> 172.21.15.92:0/95102489 conn(0x7f795009b590 msgr2=0x7f795012a430 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:58.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.759+0000 7f7955553640 1 -- 172.21.15.92:0/95102489 shutdown_connections 2024-09-21T09:41:58.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:58.759+0000 7f7955553640 1 -- 172.21.15.92:0/95102489 wait complete. 2024-09-21T09:41:58.764 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640260 got 47244640259 for osd.1 2024-09-21T09:41:58.772 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640262 got 47244640261 for osd.2 2024-09-21T09:41:59.739 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T09:41:59.754 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T09:41:59.765 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T09:41:59.773 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T09:41:59.860 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.858+0000 7f8f725d4640 1 -- 172.21.15.92:0/3850971574 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f8f6c12c130 msgr2=0x7f8f6c12c550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:59.860 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.858+0000 7f8f725d4640 1 --2- 172.21.15.92:0/3850971574 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f8f6c12c130 0x7f8f6c12c550 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f8f64009be0 tx=0x7f8f6402a760 comp rx=0 tx=0).stop 2024-09-21T09:41:59.860 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.858+0000 7f8f725d4640 1 -- 172.21.15.92:0/3850971574 shutdown_connections 2024-09-21T09:41:59.860 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.858+0000 7f8f725d4640 1 --2- 172.21.15.92:0/3850971574 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f8f6c12c130 0x7f8f6c12c550 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:59.860 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.858+0000 7f8f725d4640 1 --2- 172.21.15.92:0/3850971574 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f8f6c095f10 0x7f8f6c0962f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:59.860 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.858+0000 7f8f725d4640 1 -- 172.21.15.92:0/3850971574 >> 172.21.15.92:0/3850971574 conn(0x7f8f6c09c590 msgr2=0x7f8f6c09c9a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:59.861 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.858+0000 7f8f725d4640 1 -- 172.21.15.92:0/3850971574 shutdown_connections 2024-09-21T09:41:59.861 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.858+0000 7f8f725d4640 1 -- 172.21.15.92:0/3850971574 wait complete. 2024-09-21T09:41:59.861 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.859+0000 7f8f725d4640 1 Processor -- start 2024-09-21T09:41:59.861 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.859+0000 7f8f725d4640 1 -- start start 2024-09-21T09:41:59.863 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.860+0000 7f8f725d4640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f8f6c095f10 0x7f8f6c1c1e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:59.863 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.860+0000 7f8f725d4640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f8f6c096830 0x7f8f6c1c23b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:59.863 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.860+0000 7f8f725d4640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f8f6c12c130 0x7f8f6c06f960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:59.863 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.860+0000 7f8f6b7fe640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f8f6c096830 0x7f8f6c1c23b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:59.863 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.860+0000 7f8f6bfff640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f8f6c095f10 0x7f8f6c1c1e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:59.863 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.860+0000 7f8f6b7fe640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f8f6c096830 0x7f8f6c1c23b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:46986/0 (socket says 172.21.15.92:46986) 2024-09-21T09:41:59.863 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.860+0000 7f8f725d4640 1 -- 172.21.15.92:0/4061625306 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f8f6c1347b0 con 0x7f8f6c12c130 2024-09-21T09:41:59.863 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.860+0000 7f8f6b7fe640 1 -- 172.21.15.92:0/4061625306 learned_addr learned my addr 172.21.15.92:0/4061625306 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:59.863 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.860+0000 7f8f725d4640 1 -- 172.21.15.92:0/4061625306 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f8f6c134630 con 0x7f8f6c095f10 2024-09-21T09:41:59.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.860+0000 7f8f725d4640 1 -- 172.21.15.92:0/4061625306 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f8f6c134930 con 0x7f8f6c096830 2024-09-21T09:41:59.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.860+0000 7f8f70b71640 1 --2- 172.21.15.92:0/4061625306 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f8f6c12c130 0x7f8f6c06f960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:59.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.861+0000 7f8f6bfff640 1 -- 172.21.15.92:0/4061625306 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f8f6c096830 msgr2=0x7f8f6c1c23b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:59.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.861+0000 7f8f6bfff640 1 --2- 172.21.15.92:0/4061625306 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f8f6c096830 0x7f8f6c1c23b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:59.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.861+0000 7f8f6bfff640 1 -- 172.21.15.92:0/4061625306 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f8f6c12c130 msgr2=0x7f8f6c06f960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:59.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.861+0000 7f8f6bfff640 1 --2- 172.21.15.92:0/4061625306 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f8f6c12c130 0x7f8f6c06f960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:59.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.861+0000 7f8f6bfff640 1 -- 172.21.15.92:0/4061625306 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8f6c073290 con 0x7f8f6c095f10 2024-09-21T09:41:59.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.861+0000 7f8f6b7fe640 1 --2- 172.21.15.92:0/4061625306 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f8f6c096830 0x7f8f6c1c23b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:59.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.861+0000 7f8f70b71640 1 --2- 172.21.15.92:0/4061625306 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f8f6c12c130 0x7f8f6c06f960 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:59.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.861+0000 7f8f6bfff640 1 --2- 172.21.15.92:0/4061625306 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f8f6c095f10 0x7f8f6c1c1e70 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f8f5c00ea20 tx=0x7f8f5c00eee0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:59.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.861+0000 7f8f68ff9640 1 -- 172.21.15.92:0/4061625306 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8f5c00ccc0 con 0x7f8f6c095f10 2024-09-21T09:41:59.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.861+0000 7f8f68ff9640 1 -- 172.21.15.92:0/4061625306 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f8f5c010430 con 0x7f8f6c095f10 2024-09-21T09:41:59.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.861+0000 7f8f68ff9640 1 -- 172.21.15.92:0/4061625306 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8f5c010840 con 0x7f8f6c095f10 2024-09-21T09:41:59.864 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.861+0000 7f8f725d4640 1 -- 172.21.15.92:0/4061625306 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8f6c072660 con 0x7f8f6c095f10 2024-09-21T09:41:59.865 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.861+0000 7f8f725d4640 1 -- 172.21.15.92:0/4061625306 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f8f6c1c28f0 con 0x7f8f6c095f10 2024-09-21T09:41:59.865 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.863+0000 7f8f725d4640 1 -- 172.21.15.92:0/4061625306 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8f6c099a00 con 0x7f8f6c095f10 2024-09-21T09:41:59.865 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.863+0000 7f8f68ff9640 1 -- 172.21.15.92:0/4061625306 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f8f5c0109e0 con 0x7f8f6c095f10 2024-09-21T09:41:59.865 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.864+0000 7f8f68ff9640 1 -- 172.21.15.92:0/4061625306 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(19..19 src has 1..19) ==== 2840+0+0 (secure 0 0 0) 0x7f8f5c066300 con 0x7f8f6c095f10 2024-09-21T09:41:59.869 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.868+0000 7f8f68ff9640 1 -- 172.21.15.92:0/4061625306 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f8f6c099a00 con 0x7f8f6c095f10 2024-09-21T09:41:59.899 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.897+0000 7f800342b640 1 -- 172.21.15.92:0/191154470 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7ffc091d90 msgr2=0x7f7ffc130270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:59.899 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.897+0000 7f800342b640 1 --2- 172.21.15.92:0/191154470 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7ffc091d90 0x7f7ffc130270 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f7fec009860 tx=0x7f7fec02a390 comp rx=0 tx=0).stop 2024-09-21T09:41:59.899 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.897+0000 7f800342b640 1 -- 172.21.15.92:0/191154470 shutdown_connections 2024-09-21T09:41:59.899 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.897+0000 7f800342b640 1 --2- 172.21.15.92:0/191154470 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7ffc092170 0x7f7ffc1327c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:59.900 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.897+0000 7f800342b640 1 --2- 172.21.15.92:0/191154470 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7ffc091d90 0x7f7ffc130270 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:59.900 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.897+0000 7f800342b640 1 -- 172.21.15.92:0/191154470 >> 172.21.15.92:0/191154470 conn(0x7f7ffc09d510 msgr2=0x7f7ffc09d920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:59.900 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.897+0000 7feb1692c640 1 -- 172.21.15.92:0/490643647 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7feb10097280 msgr2=0x7feb10091680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:59.900 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.897+0000 7f6c4b66b640 1 -- 172.21.15.92:0/420622363 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6c4408c780 msgr2=0x7f6c4408cb60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:59.900 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.897+0000 7feb1692c640 1 --2- 172.21.15.92:0/490643647 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7feb10097280 0x7feb10091680 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7feb00005fd0 tx=0x7feb0002a410 comp rx=0 tx=0).stop 2024-09-21T09:41:59.901 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.897+0000 7f800342b640 1 -- 172.21.15.92:0/191154470 shutdown_connections 2024-09-21T09:41:59.901 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.897+0000 7f800342b640 1 -- 172.21.15.92:0/191154470 wait complete. 2024-09-21T09:41:59.901 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.897+0000 7f6c4b66b640 1 --2- 172.21.15.92:0/420622363 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6c4408c780 0x7f6c4408cb60 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f6c40005fd0 tx=0x7f6c4002a4b0 comp rx=0 tx=0).stop 2024-09-21T09:41:59.901 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.897+0000 7feb1692c640 1 -- 172.21.15.92:0/490643647 shutdown_connections 2024-09-21T09:41:59.901 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.898+0000 7feb1692c640 1 --2- 172.21.15.92:0/490643647 >> v2:172.21.15.92:3300/0 conn(0x7feb1006bbc0 0x7feb1006c050 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:59.901 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.898+0000 7feb1692c640 1 --2- 172.21.15.92:0/490643647 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7feb10091bc0 0x7feb10091fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:59.902 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.898+0000 7feb1692c640 1 --2- 172.21.15.92:0/490643647 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7feb10097280 0x7feb10091680 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:59.902 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.898+0000 7feb1692c640 1 -- 172.21.15.92:0/490643647 >> 172.21.15.92:0/490643647 conn(0x7feb1009d510 msgr2=0x7feb1009d920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:59.902 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.898+0000 7f6c4b66b640 1 -- 172.21.15.92:0/420622363 shutdown_connections 2024-09-21T09:41:59.902 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.898+0000 7f6c4b66b640 1 --2- 172.21.15.92:0/420622363 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6c4412e3a0 0x7f6c4412e7d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:59.902 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.898+0000 7f6c4b66b640 1 --2- 172.21.15.92:0/420622363 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6c4408c780 0x7f6c4408cb60 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:59.902 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.898+0000 7f6c4b66b640 1 -- 172.21.15.92:0/420622363 >> 172.21.15.92:0/420622363 conn(0x7f6c440a0c80 msgr2=0x7f6c4408b000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:41:59.902 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.898+0000 7feb1692c640 1 -- 172.21.15.92:0/490643647 shutdown_connections 2024-09-21T09:41:59.903 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.898+0000 7feb1692c640 1 -- 172.21.15.92:0/490643647 wait complete. 2024-09-21T09:41:59.903 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.898+0000 7f6c4b66b640 1 -- 172.21.15.92:0/420622363 shutdown_connections 2024-09-21T09:41:59.903 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.898+0000 7f6c4b66b640 1 -- 172.21.15.92:0/420622363 wait complete. 2024-09-21T09:41:59.903 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.898+0000 7f800342b640 1 Processor -- start 2024-09-21T09:41:59.903 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.898+0000 7f800342b640 1 -- start start 2024-09-21T09:41:59.904 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.898+0000 7feb1692c640 1 Processor -- start 2024-09-21T09:41:59.904 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7feb1692c640 1 -- start start 2024-09-21T09:41:59.905 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7f800342b640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7ffc091d90 0x7f7ffc08f480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:59.905 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7f800342b640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7ffc092170 0x7f7ffc08f9c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:59.905 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7f80011c7640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7ffc091d90 0x7f7ffc08f480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:59.905 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7f80009c6640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7ffc092170 0x7f7ffc08f9c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:59.905 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7f80011c7640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7ffc091d90 0x7f7ffc08f480 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:41118/0 (socket says 172.21.15.92:41118) 2024-09-21T09:41:59.905 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7f80009c6640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7ffc092170 0x7f7ffc08f9c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:40502/0 (socket says 172.21.15.92:40502) 2024-09-21T09:41:59.905 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7f800342b640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7ffc097280 0x7f7ffc08ff00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:59.905 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7f6c4b66b640 1 Processor -- start 2024-09-21T09:41:59.906 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7f80009c6640 1 -- 172.21.15.92:0/772533369 learned_addr learned my addr 172.21.15.92:0/772533369 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:59.906 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7f800342b640 1 -- 172.21.15.92:0/772533369 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f7ffc06c700 con 0x7f7ffc092170 2024-09-21T09:41:59.906 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7f6c4b66b640 1 -- start start 2024-09-21T09:41:59.907 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7f800342b640 1 -- 172.21.15.92:0/772533369 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f7ffc06c580 con 0x7f7ffc091d90 2024-09-21T09:41:59.907 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7f800342b640 1 -- 172.21.15.92:0/772533369 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f7ffc06c880 con 0x7f7ffc097280 2024-09-21T09:41:59.907 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7feb1692c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7feb1006bbc0 0x7feb101c1e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:59.907 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7f80019c8640 1 --2- 172.21.15.92:0/772533369 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7ffc097280 0x7f7ffc08ff00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:59.907 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7f80009c6640 1 -- 172.21.15.92:0/772533369 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7ffc097280 msgr2=0x7f7ffc08ff00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:59.907 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7f80009c6640 1 --2- 172.21.15.92:0/772533369 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7ffc097280 0x7f7ffc08ff00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:59.908 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7f80009c6640 1 -- 172.21.15.92:0/772533369 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7ffc091d90 msgr2=0x7f7ffc08f480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:59.908 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7f80009c6640 1 --2- 172.21.15.92:0/772533369 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7ffc091d90 0x7f7ffc08f480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:59.908 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7f80009c6640 1 -- 172.21.15.92:0/772533369 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7ffc090d80 con 0x7f7ffc092170 2024-09-21T09:41:59.908 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7feb1692c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7feb10091bc0 0x7feb101c2390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:59.908 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.899+0000 7feb1692c640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7feb10132d60 0x7feb101312f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:59.908 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7feb1692c640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7feb10133810 con 0x7feb1006bbc0 2024-09-21T09:41:59.909 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7f80009c6640 1 --2- 172.21.15.92:0/772533369 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7ffc092170 0x7f7ffc08f9c0 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f7fec000c00 tx=0x7f7fec033af0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:59.909 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7feb1692c640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7feb10133690 con 0x7feb10091bc0 2024-09-21T09:41:59.909 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7feb1692c640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7feb10133990 con 0x7feb10132d60 2024-09-21T09:41:59.909 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7feb14ec9640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7feb10132d60 0x7feb101312f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:59.909 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7f7ff1ffb640 1 -- 172.21.15.92:0/772533369 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7fec037050 con 0x7f7ffc092170 2024-09-21T09:41:59.910 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7f6c4b66b640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6c4408c780 0x7f6c4412ed60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:59.910 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7feb0ffff640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7feb1006bbc0 0x7feb101c1e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:59.910 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7feb0ffff640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7feb1006bbc0 0x7feb101c1e50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:40516/0 (socket says 172.21.15.92:40516) 2024-09-21T09:41:59.910 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7feb14ec9640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7feb10132d60 0x7feb101312f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:47038/0 (socket says 172.21.15.92:47038) 2024-09-21T09:41:59.910 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7feb0ffff640 1 -- 172.21.15.92:0/82354230 learned_addr learned my addr 172.21.15.92:0/82354230 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:59.910 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7feb0f7fe640 1 --2- 172.21.15.92:0/82354230 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7feb10091bc0 0x7feb101c2390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:59.910 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7feb14ec9640 1 -- 172.21.15.92:0/82354230 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7feb10091bc0 msgr2=0x7feb101c2390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:59.911 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7feb14ec9640 1 --2- 172.21.15.92:0/82354230 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7feb10091bc0 0x7feb101c2390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:59.911 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7feb14ec9640 1 -- 172.21.15.92:0/82354230 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7feb1006bbc0 msgr2=0x7feb101c1e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:59.911 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7feb14ec9640 1 --2- 172.21.15.92:0/82354230 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7feb1006bbc0 0x7feb101c1e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:59.911 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7f7ff1ffb640 1 -- 172.21.15.92:0/772533369 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f7fec043070 con 0x7f7ffc092170 2024-09-21T09:41:59.911 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7f7ff1ffb640 1 -- 172.21.15.92:0/772533369 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7fec003850 con 0x7f7ffc092170 2024-09-21T09:41:59.911 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7f6c4b66b640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6c4408d0a0 0x7f6c4412f2a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:59.912 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7feb14ec9640 1 -- 172.21.15.92:0/82354230 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7feb1008b110 con 0x7feb10132d60 2024-09-21T09:41:59.912 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7feb0f7fe640 1 --2- 172.21.15.92:0/82354230 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7feb10091bc0 0x7feb101c2390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T09:41:59.912 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7feb0ffff640 1 --2- 172.21.15.92:0/82354230 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7feb1006bbc0 0x7feb101c1e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:59.912 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7f800342b640 1 -- 172.21.15.92:0/772533369 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7ffc090440 con 0x7f7ffc092170 2024-09-21T09:41:59.912 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7f6c49407640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6c4408c780 0x7f6c4412ed60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:59.913 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7f6c49407640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6c4408c780 0x7f6c4412ed60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:41138/0 (socket says 172.21.15.92:41138) 2024-09-21T09:41:59.913 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7feb14ec9640 1 --2- 172.21.15.92:0/82354230 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7feb10132d60 0x7feb101312f0 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7feb08009fd0 tx=0x7feb0800ef10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:59.913 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7f6c49407640 1 -- 172.21.15.92:0/2104406507 learned_addr learned my addr 172.21.15.92:0/2104406507 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:41:59.913 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7feb0cff9640 1 -- 172.21.15.92:0/82354230 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feb0800ce20 con 0x7feb10132d60 2024-09-21T09:41:59.914 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.900+0000 7feb1692c640 1 -- 172.21.15.92:0/82354230 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7feb1008b910 con 0x7feb10132d60 2024-09-21T09:41:59.914 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.901+0000 7f800342b640 1 -- 172.21.15.92:0/772533369 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7ffc123020 con 0x7f7ffc092170 2024-09-21T09:41:59.915 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.901+0000 7f6c48c06640 1 --2- 172.21.15.92:0/2104406507 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6c4408d0a0 0x7f6c4412f2a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:59.915 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.901+0000 7f6c4b66b640 1 --2- 172.21.15.92:0/2104406507 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6c4412e3a0 0x7f6c4412f7e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:41:59.915 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.901+0000 7f6c4b66b640 1 -- 172.21.15.92:0/2104406507 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f6c44136af0 con 0x7f6c4412e3a0 2024-09-21T09:41:59.915 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.901+0000 7f6c4b66b640 1 -- 172.21.15.92:0/2104406507 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f6c44136970 con 0x7f6c4408c780 2024-09-21T09:41:59.915 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.901+0000 7f6c4b66b640 1 -- 172.21.15.92:0/2104406507 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f6c44136c70 con 0x7f6c4408d0a0 2024-09-21T09:41:59.915 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.901+0000 7f6c49c08640 1 --2- 172.21.15.92:0/2104406507 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6c4412e3a0 0x7f6c4412f7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:41:59.915 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.901+0000 7f6c49407640 1 -- 172.21.15.92:0/2104406507 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6c4408d0a0 msgr2=0x7f6c4412f2a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:59.915 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.901+0000 7feb0cff9640 1 -- 172.21.15.92:0/82354230 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7feb08004c10 con 0x7feb10132d60 2024-09-21T09:41:59.916 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.901+0000 7feb0cff9640 1 -- 172.21.15.92:0/82354230 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feb08005020 con 0x7feb10132d60 2024-09-21T09:41:59.916 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.902+0000 7f6c49407640 1 --2- 172.21.15.92:0/2104406507 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6c4408d0a0 0x7f6c4412f2a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:59.916 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.902+0000 7feb1692c640 1 -- 172.21.15.92:0/82354230 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7feb10131830 con 0x7feb10132d60 2024-09-21T09:41:59.917 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.902+0000 7f6c49407640 1 -- 172.21.15.92:0/2104406507 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6c4412e3a0 msgr2=0x7f6c4412f7e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:41:59.917 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.902+0000 7f6c49407640 1 --2- 172.21.15.92:0/2104406507 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6c4412e3a0 0x7f6c4412f7e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:41:59.917 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.902+0000 7f6c49407640 1 -- 172.21.15.92:0/2104406507 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6c440919a0 con 0x7f6c4408c780 2024-09-21T09:41:59.917 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.902+0000 7f6c48c06640 1 --2- 172.21.15.92:0/2104406507 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6c4408d0a0 0x7f6c4412f2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:59.917 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.902+0000 7f6c49c08640 1 --2- 172.21.15.92:0/2104406507 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6c4412e3a0 0x7f6c4412f7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:41:59.917 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.902+0000 7f800342b640 1 -- 172.21.15.92:0/772533369 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7ffc091680 con 0x7f7ffc092170 2024-09-21T09:41:59.917 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.902+0000 7feb0cff9640 1 -- 172.21.15.92:0/82354230 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7feb080051c0 con 0x7feb10132d60 2024-09-21T09:41:59.918 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.902+0000 7f6c49407640 1 --2- 172.21.15.92:0/2104406507 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6c4408c780 0x7f6c4412ed60 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f6c40000c00 tx=0x7f6c40003820 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:41:59.918 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.902+0000 7f7ff1ffb640 1 -- 172.21.15.92:0/772533369 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f7fec0039f0 con 0x7f7ffc092170 2024-09-21T09:41:59.918 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.902+0000 7feb0cff9640 1 -- 172.21.15.92:0/82354230 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(19..19 src has 1..19) ==== 2840+0+0 (secure 0 0 0) 0x7feb08063390 con 0x7feb10132d60 2024-09-21T09:41:59.919 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.902+0000 7f6c39ffb640 1 -- 172.21.15.92:0/2104406507 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6c4003e030 con 0x7f6c4408c780 2024-09-21T09:41:59.919 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.903+0000 7f6c39ffb640 1 -- 172.21.15.92:0/2104406507 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f6c40003d10 con 0x7f6c4408c780 2024-09-21T09:41:59.919 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.903+0000 7f6c4b66b640 1 -- 172.21.15.92:0/2104406507 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6c44128740 con 0x7f6c4408c780 2024-09-21T09:41:59.919 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.903+0000 7f7ff1ffb640 1 -- 172.21.15.92:0/772533369 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 2840+0+0 (secure 0 0 0) 0x7f7fec07ded0 con 0x7f7ffc092170 2024-09-21T09:41:59.919 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.903+0000 7feb1692c640 1 -- 172.21.15.92:0/82354230 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7feb1006c050 con 0x7feb10132d60 2024-09-21T09:41:59.920 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.903+0000 7f6c39ffb640 1 -- 172.21.15.92:0/2104406507 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6c40035040 con 0x7f6c4408c780 2024-09-21T09:41:59.920 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.903+0000 7f6c4b66b640 1 -- 172.21.15.92:0/2104406507 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f6c4412fd20 con 0x7f6c4408c780 2024-09-21T09:41:59.920 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.904+0000 7f6c39ffb640 1 -- 172.21.15.92:0/2104406507 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f6c40041030 con 0x7f6c4408c780 2024-09-21T09:41:59.920 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.904+0000 7f6c4b66b640 1 -- 172.21.15.92:0/2104406507 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6c4406ec10 con 0x7f6c4408c780 2024-09-21T09:41:59.920 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.905+0000 7f6c39ffb640 1 -- 172.21.15.92:0/2104406507 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(19..19 src has 1..19) ==== 2840+0+0 (secure 0 0 0) 0x7f6c4007e460 con 0x7f6c4408c780 2024-09-21T09:41:59.921 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.907+0000 7f7ff1ffb640 1 -- 172.21.15.92:0/772533369 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f7fec04c020 con 0x7f7ffc092170 2024-09-21T09:41:59.921 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.908+0000 7feb0cff9640 1 -- 172.21.15.92:0/82354230 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7feb0803ceb0 con 0x7feb10132d60 2024-09-21T09:41:59.921 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:41:59.909+0000 7f6c39ffb640 1 -- 172.21.15.92:0/2104406507 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f6c4406ec10 con 0x7f6c4408c780 2024-09-21T09:42:00.047 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.045+0000 7f8f725d4640 1 -- 172.21.15.92:0/4061625306 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7f8f6c0962f0 con 0x7f8f6c095f10 2024-09-21T09:42:00.047 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.046+0000 7f8f68ff9640 1 -- 172.21.15.92:0/4061625306 <== mon.1 v2:172.21.15.92:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f8f6c073290 con 0x7f8f6c095f10 2024-09-21T09:42:00.048 INFO:teuthology.orchestra.run.smithi092.stdout:47244640262 2024-09-21T09:42:00.050 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.048+0000 7f8f725d4640 1 -- 172.21.15.92:0/4061625306 >> v1:172.21.15.92:6800/1280404317 conn(0x7f8f58047fd0 legacy=0x7f8f5804a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.050 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.048+0000 7f8f725d4640 1 -- 172.21.15.92:0/4061625306 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f8f6c095f10 msgr2=0x7f8f6c1c1e70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.050 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.048+0000 7f8f725d4640 1 --2- 172.21.15.92:0/4061625306 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f8f6c095f10 0x7f8f6c1c1e70 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f8f5c00ea20 tx=0x7f8f5c00eee0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.051 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.048+0000 7f8f725d4640 1 -- 172.21.15.92:0/4061625306 shutdown_connections 2024-09-21T09:42:00.051 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.048+0000 7f8f725d4640 1 --2- 172.21.15.92:0/4061625306 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f8f6c12c130 0x7f8f6c06f960 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.051 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.048+0000 7f8f725d4640 1 --2- 172.21.15.92:0/4061625306 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f8f6c096830 0x7f8f6c1c23b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.051 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.048+0000 7f8f725d4640 1 --2- 172.21.15.92:0/4061625306 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f8f6c095f10 0x7f8f6c1c1e70 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.051 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.048+0000 7f8f725d4640 1 -- 172.21.15.92:0/4061625306 >> 172.21.15.92:0/4061625306 conn(0x7f8f6c09c590 msgr2=0x7f8f6c099e10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:42:00.051 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.049+0000 7f8f725d4640 1 -- 172.21.15.92:0/4061625306 shutdown_connections 2024-09-21T09:42:00.051 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.049+0000 7f8f725d4640 1 -- 172.21.15.92:0/4061625306 wait complete. 2024-09-21T09:42:00.058 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.056+0000 7f800342b640 1 -- 172.21.15.92:0/772533369 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7f7ffc073a60 con 0x7f7ffc092170 2024-09-21T09:42:00.058 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.056+0000 7f7ff1ffb640 1 -- 172.21.15.92:0/772533369 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f7ffc073a60 con 0x7f7ffc092170 2024-09-21T09:42:00.058 INFO:teuthology.orchestra.run.smithi092.stdout:47244640260 2024-09-21T09:42:00.060 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.058+0000 7feb1692c640 1 -- 172.21.15.92:0/82354230 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7feb1008c520 con 0x7feb10132d60 2024-09-21T09:42:00.061 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.058+0000 7f800342b640 1 -- 172.21.15.92:0/772533369 >> v1:172.21.15.92:6800/1280404317 conn(0x7f7fe0047fd0 legacy=0x7f7fe004a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.061 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.058+0000 7f800342b640 1 -- 172.21.15.92:0/772533369 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7ffc092170 msgr2=0x7f7ffc08f9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.061 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.058+0000 7f800342b640 1 --2- 172.21.15.92:0/772533369 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7ffc092170 0x7f7ffc08f9c0 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f7fec000c00 tx=0x7f7fec033af0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.061 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.058+0000 7f800342b640 1 -- 172.21.15.92:0/772533369 shutdown_connections 2024-09-21T09:42:00.061 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.058+0000 7f800342b640 1 --2- 172.21.15.92:0/772533369 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7ffc097280 0x7f7ffc08ff00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.061 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.058+0000 7f800342b640 1 --2- 172.21.15.92:0/772533369 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7ffc092170 0x7f7ffc08f9c0 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.061 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.058+0000 7f800342b640 1 --2- 172.21.15.92:0/772533369 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7ffc091d90 0x7f7ffc08f480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.061 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.058+0000 7f800342b640 1 -- 172.21.15.92:0/772533369 >> 172.21.15.92:0/772533369 conn(0x7f7ffc09d510 msgr2=0x7f7ffc0707c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:42:00.061 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.059+0000 7f800342b640 1 -- 172.21.15.92:0/772533369 shutdown_connections 2024-09-21T09:42:00.061 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.059+0000 7f800342b640 1 -- 172.21.15.92:0/772533369 wait complete. 2024-09-21T09:42:00.062 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.059+0000 7feb0cff9640 1 -- 172.21.15.92:0/82354230 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7feb0803d110 con 0x7feb10132d60 2024-09-21T09:42:00.062 INFO:teuthology.orchestra.run.smithi092.stdout:38654705669 2024-09-21T09:42:00.063 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.061+0000 7feb1692c640 1 -- 172.21.15.92:0/82354230 >> v1:172.21.15.92:6800/1280404317 conn(0x7feafc047fd0 legacy=0x7feafc04a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.063 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.061+0000 7feb1692c640 1 -- 172.21.15.92:0/82354230 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7feb10132d60 msgr2=0x7feb101312f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.063 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.061+0000 7feb1692c640 1 --2- 172.21.15.92:0/82354230 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7feb10132d60 0x7feb101312f0 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7feb08009fd0 tx=0x7feb0800ef10 comp rx=0 tx=0).stop 2024-09-21T09:42:00.063 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.061+0000 7feb1692c640 1 -- 172.21.15.92:0/82354230 shutdown_connections 2024-09-21T09:42:00.063 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.061+0000 7feb1692c640 1 --2- 172.21.15.92:0/82354230 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7feb10132d60 0x7feb101312f0 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.063 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.061+0000 7feb1692c640 1 --2- 172.21.15.92:0/82354230 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7feb10091bc0 0x7feb101c2390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.063 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.061+0000 7feb1692c640 1 --2- 172.21.15.92:0/82354230 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7feb1006bbc0 0x7feb101c1e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.063 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.061+0000 7feb1692c640 1 -- 172.21.15.92:0/82354230 >> 172.21.15.92:0/82354230 conn(0x7feb1009d510 msgr2=0x7feb100707c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:42:00.063 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.061+0000 7feb1692c640 1 -- 172.21.15.92:0/82354230 shutdown_connections 2024-09-21T09:42:00.063 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.061+0000 7feb1692c640 1 -- 172.21.15.92:0/82354230 wait complete. 2024-09-21T09:42:00.065 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640262 got 47244640262 for osd.0 2024-09-21T09:42:00.065 DEBUG:teuthology.parallel:result is None 2024-09-21T09:42:00.073 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640260 got 47244640260 for osd.1 2024-09-21T09:42:00.073 DEBUG:teuthology.parallel:result is None 2024-09-21T09:42:00.074 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705669 got 38654705669 for osd.3 2024-09-21T09:42:00.074 DEBUG:teuthology.parallel:result is None 2024-09-21T09:42:00.091 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.089+0000 7f6c4b66b640 1 -- 172.21.15.92:0/2104406507 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7f6c4408cb60 con 0x7f6c4408c780 2024-09-21T09:42:00.091 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.090+0000 7f6c39ffb640 1 -- 172.21.15.92:0/2104406507 <== mon.1 v2:172.21.15.92:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f6c4007d4b0 con 0x7f6c4408c780 2024-09-21T09:42:00.092 INFO:teuthology.orchestra.run.smithi092.stdout:47244640262 2024-09-21T09:42:00.094 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.092+0000 7f6c4b66b640 1 -- 172.21.15.92:0/2104406507 >> v1:172.21.15.92:6800/1280404317 conn(0x7f6c28047fd0 legacy=0x7f6c2804a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.094 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.092+0000 7f6c4b66b640 1 -- 172.21.15.92:0/2104406507 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6c4408c780 msgr2=0x7f6c4412ed60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.094 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.092+0000 7f6c4b66b640 1 --2- 172.21.15.92:0/2104406507 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6c4408c780 0x7f6c4412ed60 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f6c40000c00 tx=0x7f6c40003820 comp rx=0 tx=0).stop 2024-09-21T09:42:00.094 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.092+0000 7f6c4b66b640 1 -- 172.21.15.92:0/2104406507 shutdown_connections 2024-09-21T09:42:00.094 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.092+0000 7f6c4b66b640 1 --2- 172.21.15.92:0/2104406507 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6c4412e3a0 0x7f6c4412f7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.094 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.092+0000 7f6c4b66b640 1 --2- 172.21.15.92:0/2104406507 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6c4408d0a0 0x7f6c4412f2a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.094 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.092+0000 7f6c4b66b640 1 --2- 172.21.15.92:0/2104406507 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6c4408c780 0x7f6c4412ed60 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.094 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.092+0000 7f6c4b66b640 1 -- 172.21.15.92:0/2104406507 >> 172.21.15.92:0/2104406507 conn(0x7f6c440a0c80 msgr2=0x7f6c4406e500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:42:00.094 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.092+0000 7f6c4b66b640 1 -- 172.21.15.92:0/2104406507 shutdown_connections 2024-09-21T09:42:00.095 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.092+0000 7f6c4b66b640 1 -- 172.21.15.92:0/2104406507 wait complete. 2024-09-21T09:42:00.104 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640262 got 47244640262 for osd.2 2024-09-21T09:42:00.104 DEBUG:teuthology.parallel:result is None 2024-09-21T09:42:00.104 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean 2024-09-21T09:42:00.104 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T09:42:00.244 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.242+0000 7f1750c38640 1 -- 172.21.15.92:0/3178203559 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f174c130670 msgr2=0x7f174c132a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.244 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.242+0000 7f1750c38640 1 --2- 172.21.15.92:0/3178203559 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f174c130670 0x7f174c132a60 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f1744009be0 tx=0x7f174402a7c0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.244 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.242+0000 7f1750c38640 1 -- 172.21.15.92:0/3178203559 shutdown_connections 2024-09-21T09:42:00.244 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.242+0000 7f1750c38640 1 --2- 172.21.15.92:0/3178203559 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f174c130670 0x7f174c132a60 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.244 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.242+0000 7f1750c38640 1 --2- 172.21.15.92:0/3178203559 >> v2:172.21.15.92:3300/0 conn(0x7f174c12dd40 0x7f174c130130 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.244 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.242+0000 7f1750c38640 1 -- 172.21.15.92:0/3178203559 >> 172.21.15.92:0/3178203559 conn(0x7f174c09a4e0 msgr2=0x7f174c09a8f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:42:00.244 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.242+0000 7f1750c38640 1 -- 172.21.15.92:0/3178203559 shutdown_connections 2024-09-21T09:42:00.245 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.242+0000 7f1750c38640 1 -- 172.21.15.92:0/3178203559 wait complete. 2024-09-21T09:42:00.247 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.243+0000 7f1750c38640 1 Processor -- start 2024-09-21T09:42:00.247 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.243+0000 7f1750c38640 1 -- start start 2024-09-21T09:42:00.247 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.243+0000 7f1750c38640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f174c12ae20 0x7f174c123b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:42:00.247 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.243+0000 7f1750c38640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f174c12dd40 0x7f174c1240b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:42:00.247 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.243+0000 7f1750c38640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f174c130670 0x7f174c1245f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:42:00.247 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.243+0000 7f1750c38640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f174c133830 con 0x7f174c12ae20 2024-09-21T09:42:00.247 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.243+0000 7f1750c38640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f174c1336b0 con 0x7f174c130670 2024-09-21T09:42:00.247 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.243+0000 7f1750c38640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f174c1339b0 con 0x7f174c12dd40 2024-09-21T09:42:00.247 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.243+0000 7f174a59c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f174c12ae20 0x7f174c123b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:42:00.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.243+0000 7f1749d9b640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f174c12dd40 0x7f174c1240b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:42:00.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.243+0000 7f174ad9d640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f174c130670 0x7f174c1245f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:42:00.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.243+0000 7f174a59c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f174c12ae20 0x7f174c123b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:40536/0 (socket says 172.21.15.92:40536) 2024-09-21T09:42:00.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.243+0000 7f174a59c640 1 -- 172.21.15.92:0/4102464800 learned_addr learned my addr 172.21.15.92:0/4102464800 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:42:00.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.244+0000 7f174a59c640 1 -- 172.21.15.92:0/4102464800 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f174c12dd40 msgr2=0x7f174c1240b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.244+0000 7f174a59c640 1 --2- 172.21.15.92:0/4102464800 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f174c12dd40 0x7f174c1240b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.244+0000 7f174a59c640 1 -- 172.21.15.92:0/4102464800 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f174c130670 msgr2=0x7f174c1245f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.244+0000 7f174a59c640 1 --2- 172.21.15.92:0/4102464800 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f174c130670 0x7f174c1245f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.244+0000 7f174a59c640 1 -- 172.21.15.92:0/4102464800 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f174c126450 con 0x7f174c12ae20 2024-09-21T09:42:00.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.244+0000 7f174ad9d640 1 --2- 172.21.15.92:0/4102464800 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f174c130670 0x7f174c1245f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:42:00.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.244+0000 7f174a59c640 1 --2- 172.21.15.92:0/4102464800 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f174c12ae20 0x7f174c123b70 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f173c001eb0 tx=0x7f173c00ebc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:42:00.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.244+0000 7f1742ffd640 1 -- 172.21.15.92:0/4102464800 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f173c015070 con 0x7f174c12ae20 2024-09-21T09:42:00.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.244+0000 7f1742ffd640 1 -- 172.21.15.92:0/4102464800 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f173c005360 con 0x7f174c12ae20 2024-09-21T09:42:00.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.244+0000 7f1750c38640 1 -- 172.21.15.92:0/4102464800 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f174c125850 con 0x7f174c12ae20 2024-09-21T09:42:00.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.244+0000 7f1742ffd640 1 -- 172.21.15.92:0/4102464800 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f173c00ca50 con 0x7f174c12ae20 2024-09-21T09:42:00.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.244+0000 7f1750c38640 1 -- 172.21.15.92:0/4102464800 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f174c070470 con 0x7f174c12ae20 2024-09-21T09:42:00.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.245+0000 7f1742ffd640 1 -- 172.21.15.92:0/4102464800 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f173c018070 con 0x7f174c12ae20 2024-09-21T09:42:00.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.245+0000 7f1750c38640 1 -- 172.21.15.92:0/4102464800 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f174c1293d0 con 0x7f174c12ae20 2024-09-21T09:42:00.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.245+0000 7f1742ffd640 1 -- 172.21.15.92:0/4102464800 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 2840+0+0 (secure 0 0 0) 0x7f173c05fb60 con 0x7f174c12ae20 2024-09-21T09:42:00.250 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.249+0000 7f1742ffd640 1 -- 172.21.15.92:0/4102464800 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f174c1293d0 con 0x7f174c12ae20 2024-09-21T09:42:00.386 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.384+0000 7f1750c38640 1 -- 172.21.15.92:0/4102464800 --> v1:172.21.15.92:6800/1280404317 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f174c1274e0 con 0x7f172c047fd0 2024-09-21T09:42:00.387 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.385+0000 7f1742ffd640 1 -- 172.21.15.92:0/4102464800 <== mgr.4146 v1:172.21.15.92:6800/1280404317 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32195 (unknown 2965378022 0 692017763) 0x7f1734009850 con 0x7f172c047fd0 2024-09-21T09:42:00.387 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:42:00.388 INFO:teuthology.orchestra.run.smithi092.stderr:dumped all 2024-09-21T09:42:00.389 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.387+0000 7f1750c38640 1 -- 172.21.15.92:0/4102464800 >> v1:172.21.15.92:6800/1280404317 conn(0x7f172c047fd0 legacy=0x7f172c04a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.389 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.387+0000 7f1750c38640 1 -- 172.21.15.92:0/4102464800 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f174c12ae20 msgr2=0x7f174c123b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.389 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.387+0000 7f1750c38640 1 --2- 172.21.15.92:0/4102464800 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f174c12ae20 0x7f174c123b70 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7f173c001eb0 tx=0x7f173c00ebc0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.389 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.387+0000 7f1750c38640 1 -- 172.21.15.92:0/4102464800 shutdown_connections 2024-09-21T09:42:00.389 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.387+0000 7f1750c38640 1 --2- 172.21.15.92:0/4102464800 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f174c130670 0x7f174c1245f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.389 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.387+0000 7f1750c38640 1 --2- 172.21.15.92:0/4102464800 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f174c12dd40 0x7f174c1240b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.389 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.387+0000 7f1750c38640 1 --2- 172.21.15.92:0/4102464800 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f174c12ae20 0x7f174c123b70 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.389 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.387+0000 7f1750c38640 1 -- 172.21.15.92:0/4102464800 >> 172.21.15.92:0/4102464800 conn(0x7f174c09a4e0 msgr2=0x7f174c129690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:42:00.390 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.387+0000 7f1750c38640 1 -- 172.21.15.92:0/4102464800 shutdown_connections 2024-09-21T09:42:00.390 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.387+0000 7f1750c38640 1 -- 172.21.15.92:0/4102464800 wait complete. 2024-09-21T09:42:00.399 INFO:teuthology.orchestra.run.smithi092.stdout:{"pg_ready":true,"pg_map":{"version":23,"stamp":"2024-09-21T09:42:00.011309+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":12,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":110300,"kb_used_data":1776,"kb_used_omap":6,"kb_used_meta":108025,"kb_avail":377377060,"statfs":{"total":386547056640,"available":386434109440,"internally_reserved":0,"allocated":1818624,"data_stored":1351926,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6375,"internal_metadata":110618393},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.000697"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-21T09:41:55.860096+0000","last_change":"2024-09-21T09:41:55.860096+0000","last_active":"2024-09-21T09:41:55.860096+0000","last_peered":"2024-09-21T09:41:55.860096+0000","last_clean":"2024-09-21T09:41:55.860096+0000","last_became_active":"2024-09-21T09:41:53.886716+0000","last_became_peered":"2024-09-21T09:41:53.886716+0000","last_unstale":"2024-09-21T09:41:55.860096+0000","last_undegraded":"2024-09-21T09:41:55.860096+0000","last_fullsized":"2024-09-21T09:41:55.860096+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:44:13.256802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084231599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-21T09:41:55.863023+0000","last_change":"2024-09-21T09:41:55.863252+0000","last_active":"2024-09-21T09:41:55.863023+0000","last_peered":"2024-09-21T09:41:55.863023+0000","last_clean":"2024-09-21T09:41:55.863023+0000","last_became_active":"2024-09-21T09:41:53.887209+0000","last_became_peered":"2024-09-21T09:41:53.887209+0000","last_unstale":"2024-09-21T09:41:55.863023+0000","last_undegraded":"2024-09-21T09:41:55.863023+0000","last_fullsized":"2024-09-21T09:41:55.863023+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:43:15.259908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049204800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-21T09:41:55.860396+0000","last_change":"2024-09-21T09:41:55.860396+0000","last_active":"2024-09-21T09:41:55.860396+0000","last_peered":"2024-09-21T09:41:55.860396+0000","last_clean":"2024-09-21T09:41:55.860396+0000","last_became_active":"2024-09-21T09:41:53.886757+0000","last_became_peered":"2024-09-21T09:41:53.886757+0000","last_unstale":"2024-09-21T09:41:55.860396+0000","last_undegraded":"2024-09-21T09:41:55.860396+0000","last_fullsized":"2024-09-21T09:41:55.860396+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:49:19.741169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080402800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-21T09:41:55.862967+0000","last_change":"2024-09-21T09:41:55.863219+0000","last_active":"2024-09-21T09:41:55.862967+0000","last_peered":"2024-09-21T09:41:55.862967+0000","last_clean":"2024-09-21T09:41:55.862967+0000","last_became_active":"2024-09-21T09:41:53.886793+0000","last_became_peered":"2024-09-21T09:41:53.886793+0000","last_unstale":"2024-09-21T09:41:55.862967+0000","last_undegraded":"2024-09-21T09:41:55.862967+0000","last_fullsized":"2024-09-21T09:41:55.862967+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:42:08.531449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000465849,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"18'2","reported_seq":21,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-21T09:41:55.862419+0000","last_change":"2024-09-21T09:41:55.859682+0000","last_active":"2024-09-21T09:41:55.862419+0000","last_peered":"2024-09-21T09:41:55.862419+0000","last_clean":"2024-09-21T09:41:55.862419+0000","last_became_active":"2024-09-21T09:41:53.840867+0000","last_became_peered":"2024-09-21T09:41:53.840867+0000","last_unstale":"2024-09-21T09:41:55.862419+0000","last_undegraded":"2024-09-21T09:41:55.862419+0000","last_fullsized":"2024-09-21T09:41:55.862419+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:47:02.641520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010637209999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-21T09:41:55.858643+0000","last_change":"2024-09-21T09:41:55.858936+0000","last_active":"2024-09-21T09:41:55.858643+0000","last_peered":"2024-09-21T09:41:55.858643+0000","last_clean":"2024-09-21T09:41:55.858643+0000","last_became_active":"2024-09-21T09:41:53.846664+0000","last_became_peered":"2024-09-21T09:41:53.846664+0000","last_unstale":"2024-09-21T09:41:55.858643+0000","last_undegraded":"2024-09-21T09:41:55.858643+0000","last_fullsized":"2024-09-21T09:41:55.858643+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:05:39.687344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049819100000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-21T09:41:55.859721+0000","last_change":"2024-09-21T09:41:55.859721+0000","last_active":"2024-09-21T09:41:55.859721+0000","last_peered":"2024-09-21T09:41:55.859721+0000","last_clean":"2024-09-21T09:41:55.859721+0000","last_became_active":"2024-09-21T09:41:53.838115+0000","last_became_peered":"2024-09-21T09:41:53.838115+0000","last_unstale":"2024-09-21T09:41:55.859721+0000","last_undegraded":"2024-09-21T09:41:55.859721+0000","last_fullsized":"2024-09-21T09:41:55.859721+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:53:01.492593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088008499999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"16'1","reported_seq":20,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-21T09:41:55.860023+0000","last_change":"2024-09-21T09:41:55.860023+0000","last_active":"2024-09-21T09:41:55.860023+0000","last_peered":"2024-09-21T09:41:55.860023+0000","last_clean":"2024-09-21T09:41:55.860023+0000","last_became_active":"2024-09-21T09:41:53.843742+0000","last_became_peered":"2024-09-21T09:41:53.843742+0000","last_unstale":"2024-09-21T09:41:55.860023+0000","last_undegraded":"2024-09-21T09:41:55.860023+0000","last_fullsized":"2024-09-21T09:41:55.860023+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:14:31.505673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010430319999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"14'97","reported_seq":145,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-21T09:41:55.862741+0000","last_change":"2024-09-21T09:41:51.820265+0000","last_active":"2024-09-21T09:41:55.862741+0000","last_peered":"2024-09-21T09:41:55.862741+0000","last_clean":"2024-09-21T09:41:55.862741+0000","last_became_active":"2024-09-21T09:41:51.819950+0000","last_became_peered":"2024-09-21T09:41:51.819950+0000","last_unstale":"2024-09-21T09:41:55.862741+0000","last_undegraded":"2024-09-21T09:41:55.862741+0000","last_fullsized":"2024-09-21T09:41:55.862741+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:50.808452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:50.808452+0000","last_clean_scrub_stamp":"2024-09-21T09:41:50.808452+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:27:36.916853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":11,"seq":47244640263,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27188,"kb_used_data":152,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344652,"statfs":{"total":96636764160,"available":96608923648,"internally_reserved":0,"allocated":155648,"data_stored":42788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":11,"seq":47244640261,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27964,"kb_used_data":736,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94343876,"statfs":{"total":96636764160,"available":96608129024,"internally_reserved":0,"allocated":753664,"data_stored":633175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":11,"seq":47244640263,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27960,"kb_used_data":732,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94343880,"statfs":{"total":96636764160,"available":96608133120,"internally_reserved":0,"allocated":749568,"data_stored":633156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":9,"seq":38654705669,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27188,"kb_used_data":156,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344652,"statfs":{"total":96636764160,"available":96608923648,"internally_reserved":0,"allocated":159744,"data_stored":42807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:42:00.400 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T09:42:00.502 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.500+0000 7faf0960d640 1 -- 172.21.15.92:0/4017007770 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7faf04095d80 msgr2=0x7faf04096160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.503 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.500+0000 7faf0960d640 1 --2- 172.21.15.92:0/4017007770 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7faf04095d80 0x7faf04096160 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7faef8009860 tx=0x7faef802a460 comp rx=0 tx=0).stop 2024-09-21T09:42:00.503 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.500+0000 7faf0960d640 1 -- 172.21.15.92:0/4017007770 shutdown_connections 2024-09-21T09:42:00.503 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.500+0000 7faf0960d640 1 --2- 172.21.15.92:0/4017007770 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7faf041302e0 0x7faf041326d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.503 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.500+0000 7faf0960d640 1 --2- 172.21.15.92:0/4017007770 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7faf04095d80 0x7faf04096160 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.503 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.500+0000 7faf0960d640 1 -- 172.21.15.92:0/4017007770 >> 172.21.15.92:0/4017007770 conn(0x7faf0409d790 msgr2=0x7faf04099c60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:42:00.503 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.500+0000 7faf0960d640 1 -- 172.21.15.92:0/4017007770 shutdown_connections 2024-09-21T09:42:00.503 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.500+0000 7faf0960d640 1 -- 172.21.15.92:0/4017007770 wait complete. 2024-09-21T09:42:00.505 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.501+0000 7faf0960d640 1 Processor -- start 2024-09-21T09:42:00.505 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.501+0000 7faf0960d640 1 -- start start 2024-09-21T09:42:00.506 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.502+0000 7faf0960d640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7faf04093eb0 0x7faf0408eb30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:42:00.506 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.502+0000 7faf0960d640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7faf04095d80 0x7faf0408f070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:42:00.506 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.502+0000 7faf0960d640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7faf041302e0 0x7faf0408f5b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:42:00.506 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.502+0000 7faf0960d640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7faf04133770 con 0x7faf041302e0 2024-09-21T09:42:00.506 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.502+0000 7faf0960d640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7faf041335f0 con 0x7faf04095d80 2024-09-21T09:42:00.506 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.502+0000 7faf0960d640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7faf041338f0 con 0x7faf04093eb0 2024-09-21T09:42:00.506 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.502+0000 7faf027fc640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7faf04095d80 0x7faf0408f070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:42:00.506 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.502+0000 7faf037fe640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7faf041302e0 0x7faf0408f5b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:42:00.506 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.502+0000 7faf027fc640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7faf04095d80 0x7faf0408f070 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:41172/0 (socket says 172.21.15.92:41172) 2024-09-21T09:42:00.506 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.502+0000 7faf027fc640 1 -- 172.21.15.92:0/1572661165 learned_addr learned my addr 172.21.15.92:0/1572661165 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:42:00.506 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.502+0000 7faf02ffd640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7faf04093eb0 0x7faf0408eb30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:42:00.506 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.502+0000 7faf037fe640 1 -- 172.21.15.92:0/1572661165 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7faf04093eb0 msgr2=0x7faf0408eb30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.507 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.502+0000 7faf037fe640 1 --2- 172.21.15.92:0/1572661165 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7faf04093eb0 0x7faf0408eb30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.507 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.502+0000 7faf037fe640 1 -- 172.21.15.92:0/1572661165 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7faf04095d80 msgr2=0x7faf0408f070 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.507 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.502+0000 7faf037fe640 1 --2- 172.21.15.92:0/1572661165 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7faf04095d80 0x7faf0408f070 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.507 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.502+0000 7faf037fe640 1 -- 172.21.15.92:0/1572661165 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7faf0408fdd0 con 0x7faf041302e0 2024-09-21T09:42:00.507 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.502+0000 7faf027fc640 1 --2- 172.21.15.92:0/1572661165 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7faf04095d80 0x7faf0408f070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:42:00.507 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.502+0000 7faf037fe640 1 --2- 172.21.15.92:0/1572661165 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7faf041302e0 0x7faf0408f5b0 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7faefc009cb0 tx=0x7faefc00b690 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:42:00.507 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.502+0000 7faeeb7fe640 1 -- 172.21.15.92:0/1572661165 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faefc009e60 con 0x7faf041302e0 2024-09-21T09:42:00.507 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.502+0000 7faeeb7fe640 1 -- 172.21.15.92:0/1572661165 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7faefc004020 con 0x7faf041302e0 2024-09-21T09:42:00.507 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.502+0000 7faf0960d640 1 -- 172.21.15.92:0/1572661165 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7faf041223a0 con 0x7faf041302e0 2024-09-21T09:42:00.507 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.502+0000 7faeeb7fe640 1 -- 172.21.15.92:0/1572661165 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faefc004470 con 0x7faf041302e0 2024-09-21T09:42:00.507 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.503+0000 7faf0960d640 1 -- 172.21.15.92:0/1572661165 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7faf0406bdd0 con 0x7faf041302e0 2024-09-21T09:42:00.507 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.504+0000 7faeeb7fe640 1 -- 172.21.15.92:0/1572661165 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7faefc0035f0 con 0x7faf041302e0 2024-09-21T09:42:00.507 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.504+0000 7faf0960d640 1 -- 172.21.15.92:0/1572661165 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7faf04096160 con 0x7faf041302e0 2024-09-21T09:42:00.507 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.504+0000 7faeeb7fe640 1 -- 172.21.15.92:0/1572661165 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 2840+0+0 (secure 0 0 0) 0x7faefc0618b0 con 0x7faf041302e0 2024-09-21T09:42:00.510 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.507+0000 7faeeb7fe640 1 -- 172.21.15.92:0/1572661165 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7faefc060a20 con 0x7faf041302e0 2024-09-21T09:42:00.643 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.641+0000 7faf0960d640 1 -- 172.21.15.92:0/1572661165 --> v1:172.21.15.92:6800/1280404317 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7faf0406cb20 con 0x7faef0048000 2024-09-21T09:42:00.644 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.642+0000 7faeeb7fe640 1 -- 172.21.15.92:0/1572661165 <== mgr.4146 v1:172.21.15.92:6800/1280404317 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32195 (unknown 2965378022 0 692017763) 0x7faf0406cb20 con 0x7faef0048000 2024-09-21T09:42:00.645 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:42:00.645 INFO:teuthology.orchestra.run.smithi092.stderr:dumped all 2024-09-21T09:42:00.646 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.644+0000 7faf0960d640 1 -- 172.21.15.92:0/1572661165 >> v1:172.21.15.92:6800/1280404317 conn(0x7faef0048000 legacy=0x7faef004a4c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.646 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.644+0000 7faf0960d640 1 -- 172.21.15.92:0/1572661165 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7faf041302e0 msgr2=0x7faf0408f5b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.646 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.644+0000 7faf0960d640 1 --2- 172.21.15.92:0/1572661165 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7faf041302e0 0x7faf0408f5b0 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7faefc009cb0 tx=0x7faefc00b690 comp rx=0 tx=0).stop 2024-09-21T09:42:00.646 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.644+0000 7faf0960d640 1 -- 172.21.15.92:0/1572661165 shutdown_connections 2024-09-21T09:42:00.647 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.644+0000 7faf0960d640 1 --2- 172.21.15.92:0/1572661165 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7faf041302e0 0x7faf0408f5b0 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.647 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.644+0000 7faf0960d640 1 --2- 172.21.15.92:0/1572661165 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7faf04095d80 0x7faf0408f070 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.647 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.644+0000 7faf0960d640 1 --2- 172.21.15.92:0/1572661165 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7faf04093eb0 0x7faf0408eb30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.647 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.644+0000 7faf0960d640 1 -- 172.21.15.92:0/1572661165 >> 172.21.15.92:0/1572661165 conn(0x7faf0409d790 msgr2=0x7faf04099160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:42:00.647 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.644+0000 7faf0960d640 1 -- 172.21.15.92:0/1572661165 shutdown_connections 2024-09-21T09:42:00.647 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.644+0000 7faf0960d640 1 -- 172.21.15.92:0/1572661165 wait complete. 2024-09-21T09:42:00.657 INFO:teuthology.orchestra.run.smithi092.stdout:{"pg_ready":true,"pg_map":{"version":23,"stamp":"2024-09-21T09:42:00.011309+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":12,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":110300,"kb_used_data":1776,"kb_used_omap":6,"kb_used_meta":108025,"kb_avail":377377060,"statfs":{"total":386547056640,"available":386434109440,"internally_reserved":0,"allocated":1818624,"data_stored":1351926,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6375,"internal_metadata":110618393},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.000697"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-21T09:41:55.860096+0000","last_change":"2024-09-21T09:41:55.860096+0000","last_active":"2024-09-21T09:41:55.860096+0000","last_peered":"2024-09-21T09:41:55.860096+0000","last_clean":"2024-09-21T09:41:55.860096+0000","last_became_active":"2024-09-21T09:41:53.886716+0000","last_became_peered":"2024-09-21T09:41:53.886716+0000","last_unstale":"2024-09-21T09:41:55.860096+0000","last_undegraded":"2024-09-21T09:41:55.860096+0000","last_fullsized":"2024-09-21T09:41:55.860096+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:44:13.256802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084231599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-21T09:41:55.863023+0000","last_change":"2024-09-21T09:41:55.863252+0000","last_active":"2024-09-21T09:41:55.863023+0000","last_peered":"2024-09-21T09:41:55.863023+0000","last_clean":"2024-09-21T09:41:55.863023+0000","last_became_active":"2024-09-21T09:41:53.887209+0000","last_became_peered":"2024-09-21T09:41:53.887209+0000","last_unstale":"2024-09-21T09:41:55.863023+0000","last_undegraded":"2024-09-21T09:41:55.863023+0000","last_fullsized":"2024-09-21T09:41:55.863023+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:43:15.259908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049204800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-21T09:41:55.860396+0000","last_change":"2024-09-21T09:41:55.860396+0000","last_active":"2024-09-21T09:41:55.860396+0000","last_peered":"2024-09-21T09:41:55.860396+0000","last_clean":"2024-09-21T09:41:55.860396+0000","last_became_active":"2024-09-21T09:41:53.886757+0000","last_became_peered":"2024-09-21T09:41:53.886757+0000","last_unstale":"2024-09-21T09:41:55.860396+0000","last_undegraded":"2024-09-21T09:41:55.860396+0000","last_fullsized":"2024-09-21T09:41:55.860396+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:49:19.741169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080402800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-21T09:41:55.862967+0000","last_change":"2024-09-21T09:41:55.863219+0000","last_active":"2024-09-21T09:41:55.862967+0000","last_peered":"2024-09-21T09:41:55.862967+0000","last_clean":"2024-09-21T09:41:55.862967+0000","last_became_active":"2024-09-21T09:41:53.886793+0000","last_became_peered":"2024-09-21T09:41:53.886793+0000","last_unstale":"2024-09-21T09:41:55.862967+0000","last_undegraded":"2024-09-21T09:41:55.862967+0000","last_fullsized":"2024-09-21T09:41:55.862967+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:42:08.531449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000465849,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"18'2","reported_seq":21,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-21T09:41:55.862419+0000","last_change":"2024-09-21T09:41:55.859682+0000","last_active":"2024-09-21T09:41:55.862419+0000","last_peered":"2024-09-21T09:41:55.862419+0000","last_clean":"2024-09-21T09:41:55.862419+0000","last_became_active":"2024-09-21T09:41:53.840867+0000","last_became_peered":"2024-09-21T09:41:53.840867+0000","last_unstale":"2024-09-21T09:41:55.862419+0000","last_undegraded":"2024-09-21T09:41:55.862419+0000","last_fullsized":"2024-09-21T09:41:55.862419+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:47:02.641520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010637209999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-21T09:41:55.858643+0000","last_change":"2024-09-21T09:41:55.858936+0000","last_active":"2024-09-21T09:41:55.858643+0000","last_peered":"2024-09-21T09:41:55.858643+0000","last_clean":"2024-09-21T09:41:55.858643+0000","last_became_active":"2024-09-21T09:41:53.846664+0000","last_became_peered":"2024-09-21T09:41:53.846664+0000","last_unstale":"2024-09-21T09:41:55.858643+0000","last_undegraded":"2024-09-21T09:41:55.858643+0000","last_fullsized":"2024-09-21T09:41:55.858643+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:05:39.687344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049819100000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-21T09:41:55.859721+0000","last_change":"2024-09-21T09:41:55.859721+0000","last_active":"2024-09-21T09:41:55.859721+0000","last_peered":"2024-09-21T09:41:55.859721+0000","last_clean":"2024-09-21T09:41:55.859721+0000","last_became_active":"2024-09-21T09:41:53.838115+0000","last_became_peered":"2024-09-21T09:41:53.838115+0000","last_unstale":"2024-09-21T09:41:55.859721+0000","last_undegraded":"2024-09-21T09:41:55.859721+0000","last_fullsized":"2024-09-21T09:41:55.859721+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:53:01.492593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088008499999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"16'1","reported_seq":20,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-21T09:41:55.860023+0000","last_change":"2024-09-21T09:41:55.860023+0000","last_active":"2024-09-21T09:41:55.860023+0000","last_peered":"2024-09-21T09:41:55.860023+0000","last_clean":"2024-09-21T09:41:55.860023+0000","last_became_active":"2024-09-21T09:41:53.843742+0000","last_became_peered":"2024-09-21T09:41:53.843742+0000","last_unstale":"2024-09-21T09:41:55.860023+0000","last_undegraded":"2024-09-21T09:41:55.860023+0000","last_fullsized":"2024-09-21T09:41:55.860023+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:14:31.505673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010430319999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"14'97","reported_seq":145,"reported_epoch":18,"state":"active+clean","last_fresh":"2024-09-21T09:41:55.862741+0000","last_change":"2024-09-21T09:41:51.820265+0000","last_active":"2024-09-21T09:41:55.862741+0000","last_peered":"2024-09-21T09:41:55.862741+0000","last_clean":"2024-09-21T09:41:55.862741+0000","last_became_active":"2024-09-21T09:41:51.819950+0000","last_became_peered":"2024-09-21T09:41:51.819950+0000","last_unstale":"2024-09-21T09:41:55.862741+0000","last_undegraded":"2024-09-21T09:41:55.862741+0000","last_fullsized":"2024-09-21T09:41:55.862741+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:50.808452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:50.808452+0000","last_clean_scrub_stamp":"2024-09-21T09:41:50.808452+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:27:36.916853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":11,"seq":47244640263,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27188,"kb_used_data":152,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344652,"statfs":{"total":96636764160,"available":96608923648,"internally_reserved":0,"allocated":155648,"data_stored":42788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":11,"seq":47244640261,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27964,"kb_used_data":736,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94343876,"statfs":{"total":96636764160,"available":96608129024,"internally_reserved":0,"allocated":753664,"data_stored":633175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":11,"seq":47244640263,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27960,"kb_used_data":732,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94343880,"statfs":{"total":96636764160,"available":96608133120,"internally_reserved":0,"allocated":749568,"data_stored":633156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":9,"seq":38654705669,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27188,"kb_used_data":156,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344652,"statfs":{"total":96636764160,"available":96608923648,"internally_reserved":0,"allocated":159744,"data_stored":42807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:42:00.657 INFO:tasks.ceph.ceph_manager.ceph:clean! 2024-09-21T09:42:00.658 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T09:42:00.658 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy 2024-09-21T09:42:00.658 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph health --format=json 2024-09-21T09:42:00.757 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.755+0000 7f84168d2640 1 -- 172.21.15.92:0/2722280697 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f8410091870 msgr2=0x7f8410091d00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.757 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.755+0000 7f84168d2640 1 --2- 172.21.15.92:0/2722280697 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f8410091870 0x7f8410091d00 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7f8408005fd0 tx=0x7f840802a780 comp rx=0 tx=0).stop 2024-09-21T09:42:00.757 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.755+0000 7f84168d2640 1 -- 172.21.15.92:0/2722280697 shutdown_connections 2024-09-21T09:42:00.757 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.755+0000 7f84168d2640 1 --2- 172.21.15.92:0/2722280697 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f8410091870 0x7f8410091d00 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.758 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.755+0000 7f84168d2640 1 --2- 172.21.15.92:0/2722280697 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f841012a4a0 0x7f841012a880 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.758 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.755+0000 7f84168d2640 1 -- 172.21.15.92:0/2722280697 >> 172.21.15.92:0/2722280697 conn(0x7f841009a610 msgr2=0x7f841009aa20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:42:00.758 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.755+0000 7f84168d2640 1 -- 172.21.15.92:0/2722280697 shutdown_connections 2024-09-21T09:42:00.758 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.755+0000 7f84168d2640 1 -- 172.21.15.92:0/2722280697 wait complete. 2024-09-21T09:42:00.760 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.756+0000 7f84168d2640 1 Processor -- start 2024-09-21T09:42:00.760 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.756+0000 7f84168d2640 1 -- start start 2024-09-21T09:42:00.760 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.756+0000 7f84168d2640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f8410091870 0x7f84100726e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:42:00.760 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.756+0000 7f84168d2640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f841012a4a0 0x7f8410072c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:42:00.760 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.756+0000 7f84168d2640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f841012adc0 0x7f8410073160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:42:00.760 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.756+0000 7f84168d2640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f841009bc50 con 0x7f841012a4a0 2024-09-21T09:42:00.760 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.756+0000 7f84168d2640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f841009bad0 con 0x7f841012adc0 2024-09-21T09:42:00.760 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.756+0000 7f84168d2640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f8410132710 con 0x7f8410091870 2024-09-21T09:42:00.760 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.757+0000 7f840ffff640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f8410091870 0x7f84100726e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:42:00.760 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.757+0000 7f840f7fe640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f841012a4a0 0x7f8410072c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:42:00.760 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.757+0000 7f840ffff640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f8410091870 0x7f84100726e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:47078/0 (socket says 172.21.15.92:47078) 2024-09-21T09:42:00.760 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.757+0000 7f8414e6f640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f841012adc0 0x7f8410073160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:42:00.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.757+0000 7f840ffff640 1 -- 172.21.15.92:0/4021737302 learned_addr learned my addr 172.21.15.92:0/4021737302 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:42:00.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.757+0000 7f840f7fe640 1 --2- 172.21.15.92:0/4021737302 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f841012a4a0 0x7f8410072c20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:40564/0 (socket says 172.21.15.92:40564) 2024-09-21T09:42:00.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.757+0000 7f840f7fe640 1 -- 172.21.15.92:0/4021737302 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f8410091870 msgr2=0x7f84100726e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.757+0000 7f840f7fe640 1 --2- 172.21.15.92:0/4021737302 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f8410091870 0x7f84100726e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.757+0000 7f840f7fe640 1 -- 172.21.15.92:0/4021737302 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f841012adc0 msgr2=0x7f8410073160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.757+0000 7f840f7fe640 1 --2- 172.21.15.92:0/4021737302 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f841012adc0 0x7f8410073160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.757+0000 7f840f7fe640 1 -- 172.21.15.92:0/4021737302 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f841012bbb0 con 0x7f841012a4a0 2024-09-21T09:42:00.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.757+0000 7f840f7fe640 1 --2- 172.21.15.92:0/4021737302 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f841012a4a0 0x7f8410072c20 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f83f800c3a0 tx=0x7f83f800c860 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:42:00.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.757+0000 7f840cff9640 1 -- 172.21.15.92:0/4021737302 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f83f8011790 con 0x7f841012a4a0 2024-09-21T09:42:00.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.757+0000 7f840cff9640 1 -- 172.21.15.92:0/4021737302 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f83f8017070 con 0x7f841012a4a0 2024-09-21T09:42:00.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.757+0000 7f84168d2640 1 -- 172.21.15.92:0/4021737302 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f84101340e0 con 0x7f841012a4a0 2024-09-21T09:42:00.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.757+0000 7f840cff9640 1 -- 172.21.15.92:0/4021737302 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f83f8011cc0 con 0x7f841012a4a0 2024-09-21T09:42:00.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.757+0000 7f84168d2640 1 -- 172.21.15.92:0/4021737302 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f841012b860 con 0x7f841012a4a0 2024-09-21T09:42:00.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.758+0000 7f84168d2640 1 -- 172.21.15.92:0/4021737302 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8410091d00 con 0x7f841012a4a0 2024-09-21T09:42:00.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.758+0000 7f840cff9640 1 -- 172.21.15.92:0/4021737302 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f83f801c460 con 0x7f841012a4a0 2024-09-21T09:42:00.762 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.758+0000 7f840cff9640 1 -- 172.21.15.92:0/4021737302 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 2840+0+0 (secure 0 0 0) 0x7f83f805d6c0 con 0x7f841012a4a0 2024-09-21T09:42:00.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.762+0000 7f840cff9640 1 -- 172.21.15.92:0/4021737302 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f83f8011e60 con 0x7f841012a4a0 2024-09-21T09:42:00.926 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.924+0000 7f84168d2640 1 -- 172.21.15.92:0/4021737302 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "health", "format": "json"} v 0) -- 0x7f8410073ee0 con 0x7f841012a4a0 2024-09-21T09:42:00.926 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.924+0000 7f840cff9640 1 -- 172.21.15.92:0/4021737302 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "health", "format": "json"}]=0 v0) ==== 72+0+46 (secure 0 0 0) 0x7f83f805c830 con 0x7f841012a4a0 2024-09-21T09:42:00.926 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:42:00.926 INFO:teuthology.orchestra.run.smithi092.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T09:42:00.928 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.926+0000 7f84168d2640 1 -- 172.21.15.92:0/4021737302 >> v1:172.21.15.92:6800/1280404317 conn(0x7f83f0047fd0 legacy=0x7f83f004a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.928 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.926+0000 7f84168d2640 1 -- 172.21.15.92:0/4021737302 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f841012a4a0 msgr2=0x7f8410072c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:00.928 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.926+0000 7f84168d2640 1 --2- 172.21.15.92:0/4021737302 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f841012a4a0 0x7f8410072c20 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f83f800c3a0 tx=0x7f83f800c860 comp rx=0 tx=0).stop 2024-09-21T09:42:00.928 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.926+0000 7f84168d2640 1 -- 172.21.15.92:0/4021737302 shutdown_connections 2024-09-21T09:42:00.928 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.926+0000 7f84168d2640 1 --2- 172.21.15.92:0/4021737302 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f841012adc0 0x7f8410073160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.928 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.926+0000 7f84168d2640 1 --2- 172.21.15.92:0/4021737302 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f841012a4a0 0x7f8410072c20 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.928 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.926+0000 7f84168d2640 1 --2- 172.21.15.92:0/4021737302 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f8410091870 0x7f84100726e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:00.928 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.926+0000 7f84168d2640 1 -- 172.21.15.92:0/4021737302 >> 172.21.15.92:0/4021737302 conn(0x7f841009a610 msgr2=0x7f8410094280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:42:00.929 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.926+0000 7f84168d2640 1 -- 172.21.15.92:0/4021737302 shutdown_connections 2024-09-21T09:42:00.929 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:00.926+0000 7f84168d2640 1 -- 172.21.15.92:0/4021737302 wait complete. 2024-09-21T09:42:00.938 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy done 2024-09-21T09:42:00.938 INFO:teuthology.run_tasks:Running task exec... 2024-09-21T09:42:00.948 INFO:teuthology.task.exec:Executing custom commands... 2024-09-21T09:42:00.948 INFO:teuthology.task.exec:Running commands on role osd.0 host ubuntu@smithi092.front.sepia.ceph.com 2024-09-21T09:42:00.948 DEBUG:teuthology.orchestra.run.smithi092:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph osd set-require-min-compat-client squid' 2024-09-21T09:42:01.046 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.044+0000 7fca795fd640 1 -- 172.21.15.92:0/1737894167 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fca7412abd0 msgr2=0x7fca7412afb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:01.047 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.044+0000 7fca795fd640 1 --2- 172.21.15.92:0/1737894167 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fca7412abd0 0x7fca7412afb0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7fca64009860 tx=0x7fca6402a4b0 comp rx=0 tx=0).stop 2024-09-21T09:42:01.047 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.044+0000 7fca795fd640 1 -- 172.21.15.92:0/1737894167 shutdown_connections 2024-09-21T09:42:01.047 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.044+0000 7fca795fd640 1 --2- 172.21.15.92:0/1737894167 >> v2:172.21.15.92:3300/0 conn(0x7fca74070f20 0x7fca74135fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:01.047 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.044+0000 7fca795fd640 1 --2- 172.21.15.92:0/1737894167 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fca7412abd0 0x7fca7412afb0 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:01.047 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.044+0000 7fca795fd640 1 -- 172.21.15.92:0/1737894167 >> 172.21.15.92:0/1737894167 conn(0x7fca7409ef70 msgr2=0x7fca7409f380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:42:01.047 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.044+0000 7fca795fd640 1 -- 172.21.15.92:0/1737894167 shutdown_connections 2024-09-21T09:42:01.047 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.044+0000 7fca795fd640 1 -- 172.21.15.92:0/1737894167 wait complete. 2024-09-21T09:42:01.050 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.045+0000 7fca795fd640 1 Processor -- start 2024-09-21T09:42:01.050 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.045+0000 7fca795fd640 1 -- start start 2024-09-21T09:42:01.050 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca795fd640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fca74070f20 0x7fca74131f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:42:01.050 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca795fd640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fca74072bb0 0x7fca74132480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:42:01.050 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca795fd640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fca7412abd0 0x7fca741329c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:42:01.050 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca795fd640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fca74136be0 con 0x7fca7412abd0 2024-09-21T09:42:01.050 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca795fd640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fca74136a60 con 0x7fca74070f20 2024-09-21T09:42:01.050 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca795fd640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fca74136d60 con 0x7fca74072bb0 2024-09-21T09:42:01.050 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca72ffd640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fca74070f20 0x7fca74131f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:42:01.050 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca737fe640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fca7412abd0 0x7fca741329c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:42:01.050 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca72ffd640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fca74070f20 0x7fca74131f40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:41196/0 (socket says 172.21.15.92:41196) 2024-09-21T09:42:01.051 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca72ffd640 1 -- 172.21.15.92:0/1043018292 learned_addr learned my addr 172.21.15.92:0/1043018292 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:42:01.051 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca727fc640 1 --2- 172.21.15.92:0/1043018292 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fca74072bb0 0x7fca74132480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:42:01.051 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca737fe640 1 -- 172.21.15.92:0/1043018292 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fca74072bb0 msgr2=0x7fca74132480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:01.051 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca737fe640 1 --2- 172.21.15.92:0/1043018292 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fca74072bb0 0x7fca74132480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:01.051 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca737fe640 1 -- 172.21.15.92:0/1043018292 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fca74070f20 msgr2=0x7fca74131f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:01.051 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca737fe640 1 --2- 172.21.15.92:0/1043018292 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fca74070f20 0x7fca74131f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:01.051 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca737fe640 1 -- 172.21.15.92:0/1043018292 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fca741343c0 con 0x7fca7412abd0 2024-09-21T09:42:01.051 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca72ffd640 1 --2- 172.21.15.92:0/1043018292 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fca74070f20 0x7fca74131f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:42:01.051 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca737fe640 1 --2- 172.21.15.92:0/1043018292 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fca7412abd0 0x7fca741329c0 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7fca6c00efc0 tx=0x7fca6c00c800 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:42:01.051 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca5b7fe640 1 -- 172.21.15.92:0/1043018292 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fca6c003970 con 0x7fca7412abd0 2024-09-21T09:42:01.051 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca5b7fe640 1 -- 172.21.15.92:0/1043018292 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fca6c003c00 con 0x7fca7412abd0 2024-09-21T09:42:01.051 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca795fd640 1 -- 172.21.15.92:0/1043018292 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fca741337c0 con 0x7fca7412abd0 2024-09-21T09:42:01.051 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca5b7fe640 1 -- 172.21.15.92:0/1043018292 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fca6c004020 con 0x7fca7412abd0 2024-09-21T09:42:01.052 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.046+0000 7fca795fd640 1 -- 172.21.15.92:0/1043018292 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fca741ca060 con 0x7fca7412abd0 2024-09-21T09:42:01.052 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.048+0000 7fca795fd640 1 -- 172.21.15.92:0/1043018292 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fca7412afb0 con 0x7fca7412abd0 2024-09-21T09:42:01.052 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.048+0000 7fca5b7fe640 1 -- 172.21.15.92:0/1043018292 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fca6c01d020 con 0x7fca7412abd0 2024-09-21T09:42:01.052 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.048+0000 7fca5b7fe640 1 -- 172.21.15.92:0/1043018292 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 2840+0+0 (secure 0 0 0) 0x7fca6c05df00 con 0x7fca7412abd0 2024-09-21T09:42:01.053 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.052+0000 7fca5b7fe640 1 -- 172.21.15.92:0/1043018292 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7fca6c05d070 con 0x7fca7412abd0 2024-09-21T09:42:01.201 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.199+0000 7fca795fd640 1 -- 172.21.15.92:0/1043018292 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd set-require-min-compat-client", "version": "squid"} v 0) -- 0x7fca74075ab0 con 0x7fca7412abd0 2024-09-21T09:42:01.459 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.457+0000 7fca5b7fe640 1 -- 172.21.15.92:0/1043018292 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd set-require-min-compat-client", "version": "squid"}]=0 set require_min_compat_client to squid v20) ==== 139+0+0 (secure 0 0 0) 0x7fca6c03d130 con 0x7fca7412abd0 2024-09-21T09:42:01.459 INFO:teuthology.orchestra.run.smithi092.stderr:set require_min_compat_client to squid 2024-09-21T09:42:01.461 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.459+0000 7fca795fd640 1 -- 172.21.15.92:0/1043018292 >> v1:172.21.15.92:6800/1280404317 conn(0x7fca60047fd0 legacy=0x7fca6004a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:01.461 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.459+0000 7fca795fd640 1 -- 172.21.15.92:0/1043018292 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fca7412abd0 msgr2=0x7fca741329c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:01.461 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.459+0000 7fca795fd640 1 --2- 172.21.15.92:0/1043018292 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fca7412abd0 0x7fca741329c0 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7fca6c00efc0 tx=0x7fca6c00c800 comp rx=0 tx=0).stop 2024-09-21T09:42:01.461 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.459+0000 7fca795fd640 1 -- 172.21.15.92:0/1043018292 shutdown_connections 2024-09-21T09:42:01.461 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.459+0000 7fca795fd640 1 --2- 172.21.15.92:0/1043018292 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fca7412abd0 0x7fca741329c0 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:01.461 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.459+0000 7fca795fd640 1 --2- 172.21.15.92:0/1043018292 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fca74072bb0 0x7fca74132480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:01.461 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.459+0000 7fca795fd640 1 --2- 172.21.15.92:0/1043018292 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fca74070f20 0x7fca74131f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:01.461 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.459+0000 7fca795fd640 1 -- 172.21.15.92:0/1043018292 >> 172.21.15.92:0/1043018292 conn(0x7fca7409ef70 msgr2=0x7fca7406f6b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:42:01.461 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.459+0000 7fca795fd640 1 -- 172.21.15.92:0/1043018292 shutdown_connections 2024-09-21T09:42:01.461 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.459+0000 7fca795fd640 1 -- 172.21.15.92:0/1043018292 wait complete. 2024-09-21T09:42:01.473 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:42:01.481 INFO:teuthology.task.print:**** done ceph 2024-09-21T09:42:01.481 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:42:01.531 INFO:teuthology.task.print:*** upgrading, no cephfs present 2024-09-21T09:42:01.531 INFO:teuthology.run_tasks:Running task exec... 2024-09-21T09:42:01.564 INFO:teuthology.task.exec:Executing custom commands... 2024-09-21T09:42:01.564 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi092.front.sepia.ceph.com 2024-09-21T09:42:01.565 DEBUG:teuthology.orchestra.run.smithi092:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs dump' 2024-09-21T09:42:01.662 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.660+0000 7f2bd7880640 1 -- 172.21.15.92:0/2723623649 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2bd012e820 msgr2=0x7f2bd01363e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:01.663 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.660+0000 7f2bd7880640 1 --2- 172.21.15.92:0/2723623649 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2bd012e820 0x7f2bd01363e0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f2bc8009be0 tx=0x7f2bc802a800 comp rx=0 tx=0).stop 2024-09-21T09:42:01.663 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.660+0000 7f2bd7880640 1 -- 172.21.15.92:0/2723623649 shutdown_connections 2024-09-21T09:42:01.663 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.660+0000 7f2bd7880640 1 --2- 172.21.15.92:0/2723623649 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2bd012e820 0x7f2bd01363e0 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:01.663 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.660+0000 7f2bd7880640 1 --2- 172.21.15.92:0/2723623649 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2bd006e940 0x7f2bd006ed20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:01.663 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.660+0000 7f2bd7880640 1 -- 172.21.15.92:0/2723623649 >> 172.21.15.92:0/2723623649 conn(0x7f2bd009c640 msgr2=0x7f2bd009ca50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:42:01.663 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.661+0000 7f2bd7880640 1 -- 172.21.15.92:0/2723623649 shutdown_connections 2024-09-21T09:42:01.663 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.661+0000 7f2bd7880640 1 -- 172.21.15.92:0/2723623649 wait complete. 2024-09-21T09:42:01.665 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.661+0000 7f2bd7880640 1 Processor -- start 2024-09-21T09:42:01.665 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.661+0000 7f2bd7880640 1 -- start start 2024-09-21T09:42:01.665 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.662+0000 7f2bd7880640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2bd006e940 0x7f2bd011bdd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:42:01.665 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.662+0000 7f2bd7880640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2bd006f260 0x7f2bd011c310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:42:01.665 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.662+0000 7f2bd7880640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2bd012e820 0x7f2bd011c850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:42:01.666 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.662+0000 7f2bd7880640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f2bd0136cb0 con 0x7f2bd006f260 2024-09-21T09:42:01.666 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.662+0000 7f2bd7880640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f2bd0136b30 con 0x7f2bd012e820 2024-09-21T09:42:01.666 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.662+0000 7f2bd7880640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f2bd0136e30 con 0x7f2bd006e940 2024-09-21T09:42:01.666 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.662+0000 7f2bd561c640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2bd006e940 0x7f2bd011bdd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:42:01.666 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.662+0000 7f2bd5e1d640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2bd012e820 0x7f2bd011c850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:42:01.666 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.662+0000 7f2bd561c640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2bd006e940 0x7f2bd011bdd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:47104/0 (socket says 172.21.15.92:47104) 2024-09-21T09:42:01.666 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.662+0000 7f2bd561c640 1 -- 172.21.15.92:0/2806815160 learned_addr learned my addr 172.21.15.92:0/2806815160 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:42:01.666 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.662+0000 7f2bd4e1b640 1 --2- 172.21.15.92:0/2806815160 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2bd006f260 0x7f2bd011c310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:42:01.666 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.662+0000 7f2bd561c640 1 -- 172.21.15.92:0/2806815160 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2bd012e820 msgr2=0x7f2bd011c850 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:01.666 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.662+0000 7f2bd561c640 1 --2- 172.21.15.92:0/2806815160 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2bd012e820 0x7f2bd011c850 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:01.666 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.662+0000 7f2bd561c640 1 -- 172.21.15.92:0/2806815160 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2bd006f260 msgr2=0x7f2bd011c310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:01.666 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.662+0000 7f2bd561c640 1 --2- 172.21.15.92:0/2806815160 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2bd006f260 0x7f2bd011c310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:01.667 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.662+0000 7f2bd561c640 1 -- 172.21.15.92:0/2806815160 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2bd011d8a0 con 0x7f2bd006e940 2024-09-21T09:42:01.667 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.662+0000 7f2bd561c640 1 --2- 172.21.15.92:0/2806815160 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2bd006e940 0x7f2bd011bdd0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f2bcc00ea60 tx=0x7f2bcc00ef20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:42:01.667 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.662+0000 7f2bc5ffb640 1 -- 172.21.15.92:0/2806815160 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2bcc00cde0 con 0x7f2bd006e940 2024-09-21T09:42:01.667 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.662+0000 7f2bc5ffb640 1 -- 172.21.15.92:0/2806815160 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2bcc004020 con 0x7f2bd006e940 2024-09-21T09:42:01.667 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.662+0000 7f2bd7880640 1 -- 172.21.15.92:0/2806815160 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2bd011e000 con 0x7f2bd006e940 2024-09-21T09:42:01.667 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.662+0000 7f2bc5ffb640 1 -- 172.21.15.92:0/2806815160 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2bcc004450 con 0x7f2bd006e940 2024-09-21T09:42:01.667 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.663+0000 7f2bd7880640 1 -- 172.21.15.92:0/2806815160 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f2bd012e370 con 0x7f2bd006e940 2024-09-21T09:42:01.667 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.664+0000 7f2bd7880640 1 -- 172.21.15.92:0/2806815160 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2bd006ed20 con 0x7f2bd006e940 2024-09-21T09:42:01.667 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.664+0000 7f2bc5ffb640 1 -- 172.21.15.92:0/2806815160 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f2bcc0035f0 con 0x7f2bd006e940 2024-09-21T09:42:01.667 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.664+0000 7f2bc5ffb640 1 -- 172.21.15.92:0/2806815160 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7f2bcc05e250 con 0x7f2bd006e940 2024-09-21T09:42:01.669 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.668+0000 7f2bc5ffb640 1 -- 172.21.15.92:0/2806815160 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+151260 (secure 0 0 0) 0x7f2bcc05d3e0 con 0x7f2bd006e940 2024-09-21T09:42:01.832 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.830+0000 7f2bd7880640 1 -- 172.21.15.92:0/2806815160 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "fs dump"} v 0) -- 0x7f2bd0134a50 con 0x7f2bd006e940 2024-09-21T09:42:01.832 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.830+0000 7f2bc5ffb640 1 -- 172.21.15.92:0/2806815160 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mon_command_ack([{"prefix": "fs dump"}]=0 dumped fsmap epoch 1 v1) ==== 75+0+440 (secure 0 0 0) 0x7f2bcc03d4c0 con 0x7f2bd006e940 2024-09-21T09:42:01.832 INFO:teuthology.orchestra.run.smithi092.stdout:e1 2024-09-21T09:42:01.832 INFO:teuthology.orchestra.run.smithi092.stdout:btime 2024-09-21T09:41:34:677977+0000 2024-09-21T09:42:01.832 INFO:teuthology.orchestra.run.smithi092.stdout:enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:42:01.833 INFO:teuthology.orchestra.run.smithi092.stdout:default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2,11=minor log segments,12=quiesce subvolumes} 2024-09-21T09:42:01.833 INFO:teuthology.orchestra.run.smithi092.stdout:legacy client fscid: -1 2024-09-21T09:42:01.833 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:42:01.833 INFO:teuthology.orchestra.run.smithi092.stdout:No filesystems configured 2024-09-21T09:42:01.834 INFO:teuthology.orchestra.run.smithi092.stderr:dumped fsmap epoch 1 2024-09-21T09:42:01.834 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.832+0000 7f2bd7880640 1 -- 172.21.15.92:0/2806815160 >> v1:172.21.15.92:6800/1280404317 conn(0x7f2bb4047fd0 legacy=0x7f2bb404a490 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:01.834 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.832+0000 7f2bd7880640 1 -- 172.21.15.92:0/2806815160 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2bd006e940 msgr2=0x7f2bd011bdd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:42:01.834 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.832+0000 7f2bd7880640 1 --2- 172.21.15.92:0/2806815160 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2bd006e940 0x7f2bd011bdd0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f2bcc00ea60 tx=0x7f2bcc00ef20 comp rx=0 tx=0).stop 2024-09-21T09:42:01.834 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.832+0000 7f2bd7880640 1 -- 172.21.15.92:0/2806815160 shutdown_connections 2024-09-21T09:42:01.834 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.832+0000 7f2bd7880640 1 --2- 172.21.15.92:0/2806815160 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2bd012e820 0x7f2bd011c850 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:01.834 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.832+0000 7f2bd7880640 1 --2- 172.21.15.92:0/2806815160 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2bd006f260 0x7f2bd011c310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:01.835 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.832+0000 7f2bd7880640 1 --2- 172.21.15.92:0/2806815160 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2bd006e940 0x7f2bd011bdd0 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:42:01.835 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.832+0000 7f2bd7880640 1 -- 172.21.15.92:0/2806815160 >> 172.21.15.92:0/2806815160 conn(0x7f2bd009c640 msgr2=0x7f2bd0131c20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:42:01.835 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.832+0000 7f2bd7880640 1 -- 172.21.15.92:0/2806815160 shutdown_connections 2024-09-21T09:42:01.835 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:42:01.832+0000 7f2bd7880640 1 -- 172.21.15.92:0/2806815160 wait complete. 2024-09-21T09:42:01.844 INFO:teuthology.run_tasks:Running task install.upgrade... 2024-09-21T09:42:01.853 INFO:teuthology.task.install:project ceph config {'mon.a': None} overrides {'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'} 2024-09-21T09:42:01.853 INFO:teuthology.task.install:remote ubuntu@smithi092.front.sepia.ceph.com config {'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'} 2024-09-21T09:42:01.853 INFO:teuthology.task.install:extra packages: [] 2024-09-21T09:42:01.858 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-21T09:42:01.858 INFO:teuthology.task.install:Upgrading ceph rpm packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd 2024-09-21T09:42:01.859 DEBUG:teuthology.orchestra.run.smithi092:> rpm -q ceph-common --qf '%{VERSION}-%{RELEASE}' 2024-09-21T09:42:01.879 INFO:teuthology.orchestra.run.smithi092.stdout:19.1.1-373.g4e45e6b4.el9 2024-09-21T09:42:01.880 INFO:teuthology.packaging:The installed version of ceph-common is 19.1.1-373.g4e45e6b4.el9 2024-09-21T09:42:01.880 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:42:02.034 INFO:teuthology.task.install:Ceph rpm upgrade from 19.1.1-373.g4e45e6b4.el9 to 19.3.0-5092.g62da54b4 2024-09-21T09:42:02.034 INFO:teuthology.task.install.rpm:Host smithi092 is: centos 9.stream x86_64 2024-09-21T09:42:02.034 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:42:02.170 INFO:teuthology.task.install.rpm:Repo base URL: https://3.chacra.ceph.com/r/ceph/main/62da54b4b83a96ca9976b7e1d0c8272a564a2208/centos/9/flavors/default/ 2024-09-21T09:42:02.171 DEBUG:teuthology.orchestra.run.smithi092:> sudo rm -f /etc/yum.repos.d/ceph.repo 2024-09-21T09:42:02.307 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/main/62da54b4b83a96ca9976b7e1d0c8272a564a2208/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/main/62da54b4b83a96ca9976b7e1d0c8272a564a2208/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/main/62da54b4b83a96ca9976b7e1d0c8272a564a2208/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-21T09:42:02.307 DEBUG:teuthology.orchestra.run.smithi092:> set -ex 2024-09-21T09:42:02.307 DEBUG:teuthology.orchestra.run.smithi092:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-21T09:42:02.332 DEBUG:teuthology.orchestra.run.smithi092:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/62da54b4b83a96ca9976b7e1d0c8272a564a2208/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-21T09:42:02.398 DEBUG:teuthology.orchestra.run.smithi092:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-21T09:42:02.462 DEBUG:teuthology.orchestra.run.smithi092:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-21T09:42:02.517 INFO:teuthology.orchestra.run.smithi092.stdout:check_obsoletes = 1 2024-09-21T09:42:02.529 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum clean all 2024-09-21T09:42:02.761 INFO:teuthology.orchestra.run.smithi092.stdout:60 files removed 2024-09-21T09:42:02.789 DEBUG:teuthology.orchestra.run.smithi092:> sudo yum -y upgrade ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd 2024-09-21T09:42:03.348 INFO:teuthology.orchestra.run.smithi092.stdout:ceph packages for x86_64 317 kB/s | 84 kB 00:00 2024-09-21T09:42:03.641 INFO:teuthology.orchestra.run.smithi092.stdout:ceph noarch packages 97 kB/s | 24 kB 00:00 2024-09-21T09:42:03.925 INFO:teuthology.orchestra.run.smithi092.stdout:ceph source packages 7.9 kB/s | 1.9 kB 00:00 2024-09-21T09:42:05.716 INFO:teuthology.orchestra.run.smithi092.stdout:CentOS Stream 9 - BaseOS 4.7 MB/s | 8.3 MB 00:01 2024-09-21T09:42:12.083 INFO:teuthology.orchestra.run.smithi092.stdout:CentOS Stream 9 - AppStream 3.9 MB/s | 20 MB 00:05 2024-09-21T09:42:16.617 INFO:teuthology.orchestra.run.smithi092.stdout:CentOS Stream 9 - CRB 9.1 MB/s | 6.5 MB 00:00 2024-09-21T09:42:18.051 INFO:teuthology.orchestra.run.smithi092.stdout:CentOS Stream 9 - Extras packages 88 kB/s | 19 kB 00:00 2024-09-21T09:42:18.780 INFO:teuthology.orchestra.run.smithi092.stdout:Extra Packages for Enterprise Linux 37 MB/s | 23 MB 00:00 2024-09-21T09:42:24.128 INFO:teuthology.orchestra.run.smithi092.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-09-21T09:42:26.251 INFO:teuthology.orchestra.run.smithi092.stderr:Package cephadm available, but not installed. 2024-09-21T09:42:26.251 INFO:teuthology.orchestra.run.smithi092.stdout:No match for argument: cephadm 2024-09-21T09:42:26.257 INFO:teuthology.orchestra.run.smithi092.stderr:Package ceph-mgr-dashboard available, but not installed. 2024-09-21T09:42:26.257 INFO:teuthology.orchestra.run.smithi092.stdout:No match for argument: ceph-mgr-dashboard 2024-09-21T09:42:26.257 INFO:teuthology.orchestra.run.smithi092.stderr:Package ceph-mgr-diskprediction-local available, but not installed. 2024-09-21T09:42:26.258 INFO:teuthology.orchestra.run.smithi092.stdout:No match for argument: ceph-mgr-diskprediction-local 2024-09-21T09:42:26.258 INFO:teuthology.orchestra.run.smithi092.stderr:Package ceph-mgr-rook available, but not installed. 2024-09-21T09:42:26.258 INFO:teuthology.orchestra.run.smithi092.stdout:No match for argument: ceph-mgr-rook 2024-09-21T09:42:26.258 INFO:teuthology.orchestra.run.smithi092.stderr:Package ceph-mgr-cephadm available, but not installed. 2024-09-21T09:42:26.259 INFO:teuthology.orchestra.run.smithi092.stdout:No match for argument: ceph-mgr-cephadm 2024-09-21T09:42:26.261 INFO:teuthology.orchestra.run.smithi092.stderr:Package ceph-volume available, but not installed. 2024-09-21T09:42:26.262 INFO:teuthology.orchestra.run.smithi092.stdout:No match for argument: ceph-volume 2024-09-21T09:42:26.350 INFO:teuthology.orchestra.run.smithi092.stdout:Dependencies resolved. 2024-09-21T09:42:26.352 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:42:26.353 INFO:teuthology.orchestra.run.smithi092.stdout: Package Arch Version Repository Size 2024-09-21T09:42:26.353 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:42:26.353 INFO:teuthology.orchestra.run.smithi092.stdout:Upgrading: 2024-09-21T09:42:26.353 INFO:teuthology.orchestra.run.smithi092.stdout: ceph x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 6.6 k 2024-09-21T09:42:26.353 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-base x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 5.5 M 2024-09-21T09:42:26.353 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-common x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 22 M 2024-09-21T09:42:26.353 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-fuse x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 900 k 2024-09-21T09:42:26.353 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-immutable-object-cache 2024-09-21T09:42:26.353 INFO:teuthology.orchestra.run.smithi092.stdout: x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 145 k 2024-09-21T09:42:26.353 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mds x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 2.4 M 2024-09-21T09:42:26.353 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mgr x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 1.5 M 2024-09-21T09:42:26.353 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mgr-modules-core noarch 2:19.3.0-5092.g62da54b4.el9 ceph-noarch 292 k 2024-09-21T09:42:26.354 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mon x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 4.8 M 2024-09-21T09:42:26.354 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-osd x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 17 M 2024-09-21T09:42:26.354 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-radosgw x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 11 M 2024-09-21T09:42:26.354 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-selinux x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 25 k 2024-09-21T09:42:26.354 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-test x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 46 M 2024-09-21T09:42:26.354 INFO:teuthology.orchestra.run.smithi092.stdout: libcephfs-devel x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 32 k 2024-09-21T09:42:26.354 INFO:teuthology.orchestra.run.smithi092.stdout: libcephfs2 x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 756 k 2024-09-21T09:42:26.354 INFO:teuthology.orchestra.run.smithi092.stdout: libcephsqlite x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 161 k 2024-09-21T09:42:26.354 INFO:teuthology.orchestra.run.smithi092.stdout: librados-devel x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 126 k 2024-09-21T09:42:26.354 INFO:teuthology.orchestra.run.smithi092.stdout: librados2 x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 3.5 M 2024-09-21T09:42:26.354 INFO:teuthology.orchestra.run.smithi092.stdout: libradosstriper1 x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 508 k 2024-09-21T09:42:26.354 INFO:teuthology.orchestra.run.smithi092.stdout: librbd1 x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 3.2 M 2024-09-21T09:42:26.354 INFO:teuthology.orchestra.run.smithi092.stdout: librgw2 x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 5.5 M 2024-09-21T09:42:26.354 INFO:teuthology.orchestra.run.smithi092.stdout: python3-ceph-argparse x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 45 k 2024-09-21T09:42:26.355 INFO:teuthology.orchestra.run.smithi092.stdout: python3-ceph-common x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 148 k 2024-09-21T09:42:26.355 INFO:teuthology.orchestra.run.smithi092.stdout: python3-cephfs x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 161 k 2024-09-21T09:42:26.355 INFO:teuthology.orchestra.run.smithi092.stdout: python3-rados x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 321 k 2024-09-21T09:42:26.355 INFO:teuthology.orchestra.run.smithi092.stdout: python3-rbd x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 305 k 2024-09-21T09:42:26.355 INFO:teuthology.orchestra.run.smithi092.stdout: python3-rgw x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 99 k 2024-09-21T09:42:26.355 INFO:teuthology.orchestra.run.smithi092.stdout: rbd-fuse x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 85 k 2024-09-21T09:42:26.355 INFO:teuthology.orchestra.run.smithi092.stdout: rbd-mirror x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 3.2 M 2024-09-21T09:42:26.355 INFO:teuthology.orchestra.run.smithi092.stdout: rbd-nbd x86_64 2:19.3.0-5092.g62da54b4.el9 ceph 170 k 2024-09-21T09:42:26.355 INFO:teuthology.orchestra.run.smithi092.stdout:Installing dependencies: 2024-09-21T09:42:26.355 INFO:teuthology.orchestra.run.smithi092.stdout: libnbd x86_64 1.20.2-2.el9 appstream 171 k 2024-09-21T09:42:26.355 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:42:26.355 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction Summary 2024-09-21T09:42:26.355 INFO:teuthology.orchestra.run.smithi092.stdout:================================================================================ 2024-09-21T09:42:26.355 INFO:teuthology.orchestra.run.smithi092.stdout:Install 1 Package 2024-09-21T09:42:26.356 INFO:teuthology.orchestra.run.smithi092.stdout:Upgrade 30 Packages 2024-09-21T09:42:26.356 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:42:26.356 INFO:teuthology.orchestra.run.smithi092.stdout:Total download size: 129 M 2024-09-21T09:42:26.356 INFO:teuthology.orchestra.run.smithi092.stdout:Downloading Packages: 2024-09-21T09:42:26.529 INFO:teuthology.orchestra.run.smithi092.stdout:(1/31): ceph-19.3.0-5092.g62da54b4.el9.x86_64.r 52 kB/s | 6.6 kB 00:00 2024-09-21T09:42:26.554 INFO:teuthology.orchestra.run.smithi092.stdout:(2/31): libnbd-1.20.2-2.el9.x86_64.rpm 1.1 MB/s | 171 kB 00:00 2024-09-21T09:42:26.813 INFO:teuthology.orchestra.run.smithi092.stdout:(3/31): ceph-base-19.3.0-5092.g62da54b4.el9.x86 13 MB/s | 5.5 MB 00:00 2024-09-21T09:42:26.863 INFO:teuthology.orchestra.run.smithi092.stdout:(4/31): ceph-immutable-object-cache-19.3.0-5092 2.8 MB/s | 145 kB 00:00 2024-09-21T09:42:26.989 INFO:teuthology.orchestra.run.smithi092.stdout:(5/31): ceph-mds-19.3.0-5092.g62da54b4.el9.x86_ 19 MB/s | 2.4 MB 00:00 2024-09-21T09:42:27.072 INFO:teuthology.orchestra.run.smithi092.stdout:(6/31): ceph-mgr-19.3.0-5092.g62da54b4.el9.x86_ 18 MB/s | 1.5 MB 00:00 2024-09-21T09:42:27.097 INFO:teuthology.orchestra.run.smithi092.stdout:(7/31): ceph-fuse-19.3.0-5092.g62da54b4.el9.x86 1.6 MB/s | 900 kB 00:00 2024-09-21T09:42:27.256 INFO:teuthology.orchestra.run.smithi092.stdout:(8/31): ceph-mon-19.3.0-5092.g62da54b4.el9.x86_ 26 MB/s | 4.8 MB 00:00 2024-09-21T09:42:27.640 INFO:teuthology.orchestra.run.smithi092.stdout:(9/31): ceph-radosgw-19.3.0-5092.g62da54b4.el9. 28 MB/s | 11 MB 00:00 2024-09-21T09:42:27.690 INFO:teuthology.orchestra.run.smithi092.stdout:(10/31): ceph-selinux-19.3.0-5092.g62da54b4.el9 505 kB/s | 25 kB 00:00 2024-09-21T09:42:28.049 INFO:teuthology.orchestra.run.smithi092.stdout:(11/31): ceph-common-19.3.0-5092.g62da54b4.el9. 14 MB/s | 22 MB 00:01 2024-09-21T09:42:28.091 INFO:teuthology.orchestra.run.smithi092.stdout:(12/31): libcephfs-devel-19.3.0-5092.g62da54b4. 768 kB/s | 32 kB 00:00 2024-09-21T09:42:28.174 INFO:teuthology.orchestra.run.smithi092.stdout:(13/31): libcephfs2-19.3.0-5092.g62da54b4.el9.x 8.9 MB/s | 756 kB 00:00 2024-09-21T09:42:28.241 INFO:teuthology.orchestra.run.smithi092.stdout:(14/31): libcephsqlite-19.3.0-5092.g62da54b4.el 2.4 MB/s | 161 kB 00:00 2024-09-21T09:42:28.300 INFO:teuthology.orchestra.run.smithi092.stdout:(15/31): librados-devel-19.3.0-5092.g62da54b4.e 2.1 MB/s | 126 kB 00:00 2024-09-21T09:42:28.525 INFO:teuthology.orchestra.run.smithi092.stdout:(16/31): librados2-19.3.0-5092.g62da54b4.el9.x8 15 MB/s | 3.5 MB 00:00 2024-09-21T09:42:28.592 INFO:teuthology.orchestra.run.smithi092.stdout:(17/31): libradosstriper1-19.3.0-5092.g62da54b4 7.5 MB/s | 508 kB 00:00 2024-09-21T09:42:28.817 INFO:teuthology.orchestra.run.smithi092.stdout:(18/31): librbd1-19.3.0-5092.g62da54b4.el9.x86_ 14 MB/s | 3.2 MB 00:00 2024-09-21T09:42:29.151 INFO:teuthology.orchestra.run.smithi092.stdout:(19/31): librgw2-19.3.0-5092.g62da54b4.el9.x86_ 17 MB/s | 5.5 MB 00:00 2024-09-21T09:42:29.201 INFO:teuthology.orchestra.run.smithi092.stdout:(20/31): python3-ceph-argparse-19.3.0-5092.g62d 907 kB/s | 45 kB 00:00 2024-09-21T09:42:29.268 INFO:teuthology.orchestra.run.smithi092.stdout:(21/31): python3-ceph-common-19.3.0-5092.g62da5 2.2 MB/s | 148 kB 00:00 2024-09-21T09:42:29.735 INFO:teuthology.orchestra.run.smithi092.stdout:(22/31): ceph-test-19.3.0-5092.g62da54b4.el9.x8 22 MB/s | 46 MB 00:02 2024-09-21T09:42:29.760 INFO:teuthology.orchestra.run.smithi092.stdout:(23/31): python3-cephfs-19.3.0-5092.g62da54b4.e 328 kB/s | 161 kB 00:00 2024-09-21T09:42:29.785 INFO:teuthology.orchestra.run.smithi092.stdout:(24/31): python3-rados-19.3.0-5092.g62da54b4.el 6.3 MB/s | 321 kB 00:00 2024-09-21T09:42:29.811 INFO:teuthology.orchestra.run.smithi092.stdout:(25/31): python3-rbd-19.3.0-5092.g62da54b4.el9. 5.9 MB/s | 305 kB 00:00 2024-09-21T09:42:29.836 INFO:teuthology.orchestra.run.smithi092.stdout:(26/31): python3-rgw-19.3.0-5092.g62da54b4.el9. 1.9 MB/s | 99 kB 00:00 2024-09-21T09:42:29.861 INFO:teuthology.orchestra.run.smithi092.stdout:(27/31): rbd-fuse-19.3.0-5092.g62da54b4.el9.x86 1.7 MB/s | 85 kB 00:00 2024-09-21T09:42:30.037 INFO:teuthology.orchestra.run.smithi092.stdout:(28/31): ceph-osd-19.3.0-5092.g62da54b4.el9.x86 5.8 MB/s | 17 MB 00:02 2024-09-21T09:42:30.062 INFO:teuthology.orchestra.run.smithi092.stdout:(29/31): rbd-nbd-19.3.0-5092.g62da54b4.el9.x86_ 849 kB/s | 170 kB 00:00 2024-09-21T09:42:30.129 INFO:teuthology.orchestra.run.smithi092.stdout:(30/31): rbd-mirror-19.3.0-5092.g62da54b4.el9.x 11 MB/s | 3.2 MB 00:00 2024-09-21T09:42:30.154 INFO:teuthology.orchestra.run.smithi092.stdout:(31/31): ceph-mgr-modules-core-19.3.0-5092.g62d 2.4 MB/s | 292 kB 00:00 2024-09-21T09:42:30.157 INFO:teuthology.orchestra.run.smithi092.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:42:30.157 INFO:teuthology.orchestra.run.smithi092.stdout:Total 34 MB/s | 129 MB 00:03 2024-09-21T09:42:30.171 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction check 2024-09-21T09:42:30.225 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction check succeeded. 2024-09-21T09:42:30.225 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction test 2024-09-21T09:42:31.090 INFO:teuthology.orchestra.run.smithi092.stdout:Transaction test succeeded. 2024-09-21T09:42:31.090 INFO:teuthology.orchestra.run.smithi092.stdout:Running transaction 2024-09-21T09:42:33.673 INFO:teuthology.orchestra.run.smithi092.stdout: Preparing : 1/1 2024-09-21T09:42:33.771 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : librados2-2:19.3.0-5092.g62da54b4.el9.x86_64 1/61 2024-09-21T09:42:33.895 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: librados2-2:19.3.0-5092.g62da54b4.el9.x86_64 1/61 2024-09-21T09:42:33.947 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : libcephfs2-2:19.3.0-5092.g62da54b4.el9.x86_64 2/61 2024-09-21T09:42:34.559 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: libcephfs2-2:19.3.0-5092.g62da54b4.el9.x86_64 2/61 2024-09-21T09:42:34.649 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : librgw2-2:19.3.0-5092.g62da54b4.el9.x86_64 3/61 2024-09-21T09:42:34.739 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: librgw2-2:19.3.0-5092.g62da54b4.el9.x86_64 3/61 2024-09-21T09:42:34.805 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : python3-rados-2:19.3.0-5092.g62da54b4.el9.x86_64 4/61 2024-09-21T09:42:35.211 INFO:teuthology.orchestra.run.smithi092.stdout: Installing : libnbd-1.20.2-2.el9.x86_64 5/61 2024-09-21T09:42:35.259 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : librbd1-2:19.3.0-5092.g62da54b4.el9.x86_64 6/61 2024-09-21T09:42:35.334 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: librbd1-2:19.3.0-5092.g62da54b4.el9.x86_64 6/61 2024-09-21T09:42:35.369 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : libcephsqlite-2:19.3.0-5092.g62da54b4.el9.x86_64 7/61 2024-09-21T09:42:35.475 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: libcephsqlite-2:19.3.0-5092.g62da54b4.el9.x86_64 7/61 2024-09-21T09:42:35.520 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : libradosstriper1-2:19.3.0-5092.g62da54b4.el9.x86_6 8/61 2024-09-21T09:42:35.587 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: libradosstriper1-2:19.3.0-5092.g62da54b4.el9.x86_6 8/61 2024-09-21T09:42:35.629 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : python3-ceph-argparse-2:19.3.0-5092.g62da54b4.el9. 9/61 2024-09-21T09:42:35.701 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : python3-cephfs-2:19.3.0-5092.g62da54b4.el9.x86_64 10/61 2024-09-21T09:42:35.750 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : python3-rbd-2:19.3.0-5092.g62da54b4.el9.x86_64 11/61 2024-09-21T09:42:35.793 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : python3-rgw-2:19.3.0-5092.g62da54b4.el9.x86_64 12/61 2024-09-21T09:42:35.961 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : librados-devel-2:19.3.0-5092.g62da54b4.el9.x86_64 13/61 2024-09-21T09:42:36.072 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : ceph-mgr-modules-core-2:19.3.0-5092.g62da54b4.el9. 14/61 2024-09-21T09:42:36.118 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : python3-ceph-common-2:19.3.0-5092.g62da54b4.el9.x8 15/61 2024-09-21T09:42:38.691 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-common-2:19.3.0-5092.g62da54b4.el9.x86_64 16/61 2024-09-21T09:42:38.753 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : ceph-common-2:19.3.0-5092.g62da54b4.el9.x86_64 16/61 2024-09-21T09:42:39.431 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-common-2:19.3.0-5092.g62da54b4.el9.x86_64 16/61 2024-09-21T09:42:39.590 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : ceph-base-2:19.3.0-5092.g62da54b4.el9.x86_64 17/61 2024-09-21T09:42:39.654 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-base-2:19.3.0-5092.g62da54b4.el9.x86_64 17/61 2024-09-21T09:42:39.707 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : ceph-selinux-2:19.3.0-5092.g62da54b4.el9.x86_64 18/61 2024-09-21T09:42:48.249 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-selinux-2:19.3.0-5092.g62da54b4.el9.x86_64 18/61 2024-09-21T09:42:48.336 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : ceph-mds-2:19.3.0-5092.g62da54b4.el9.x86_64 19/61 2024-09-21T09:42:48.479 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-mds-2:19.3.0-5092.g62da54b4.el9.x86_64 19/61 2024-09-21T09:42:48.557 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : ceph-mgr-2:19.3.0-5092.g62da54b4.el9.x86_64 20/61 2024-09-21T09:42:49.115 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-mgr-2:19.3.0-5092.g62da54b4.el9.x86_64 20/61 2024-09-21T09:42:49.172 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : ceph-mon-2:19.3.0-5092.g62da54b4.el9.x86_64 21/61 2024-09-21T09:42:50.984 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-mon-2:19.3.0-5092.g62da54b4.el9.x86_64 21/61 2024-09-21T09:42:51.041 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : ceph-osd-2:19.3.0-5092.g62da54b4.el9.x86_64 22/61 2024-09-21T09:42:51.053 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-osd-2:19.3.0-5092.g62da54b4.el9.x86_64 22/61 2024-09-21T09:42:52.282 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : ceph-2:19.3.0-5092.g62da54b4.el9.x86_64 23/61 2024-09-21T09:42:52.350 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : ceph-radosgw-2:19.3.0-5092.g62da54b4.el9.x86_64 24/61 2024-09-21T09:42:52.372 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-radosgw-2:19.3.0-5092.g62da54b4.el9.x86_64 24/61 2024-09-21T09:42:52.418 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : ceph-immutable-object-cache-2:19.3.0-5092.g62da54b 25/61 2024-09-21T09:42:52.797 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.3.0-5092.g62da54b 25/61 2024-09-21T09:42:52.835 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : rbd-mirror-2:19.3.0-5092.g62da54b4.el9.x86_64 26/61 2024-09-21T09:42:58.629 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: rbd-mirror-2:19.3.0-5092.g62da54b4.el9.x86_64 26/61 2024-09-21T09:42:58.799 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : ceph-test-2:19.3.0-5092.g62da54b4.el9.x86_64 27/61 2024-09-21T09:42:58.869 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : libcephfs-devel-2:19.3.0-5092.g62da54b4.el9.x86_64 28/61 2024-09-21T09:42:58.927 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : rbd-fuse-2:19.3.0-5092.g62da54b4.el9.x86_64 29/61 2024-09-21T09:42:59.057 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : rbd-nbd-2:19.3.0-5092.g62da54b4.el9.x86_64 30/61 2024-09-21T09:42:59.113 INFO:teuthology.orchestra.run.smithi092.stdout: Upgrading : ceph-fuse-2:19.3.0-5092.g62da54b4.el9.x86_64 31/61 2024-09-21T09:42:59.145 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : ceph-2:19.1.1-373.g4e45e6b4.el9.x86_64 32/61 2024-09-21T09:42:59.184 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : ceph-test-2:19.1.1-373.g4e45e6b4.el9.x86_64 33/61 2024-09-21T09:42:59.189 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-radosgw-2:19.1.1-373.g4e45e6b4.el9.x86_64 34/61 2024-09-21T09:42:59.189 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : ceph-radosgw-2:19.1.1-373.g4e45e6b4.el9.x86_64 34/61 2024-09-21T09:42:59.235 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-radosgw-2:19.1.1-373.g4e45e6b4.el9.x86_64 34/61 2024-09-21T09:42:59.259 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : rbd-nbd-2:19.1.1-373.g4e45e6b4.el9.x86_64 35/61 2024-09-21T09:42:59.264 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: rbd-mirror-2:19.1.1-373.g4e45e6b4.el9.x86_64 36/61 2024-09-21T09:42:59.264 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : rbd-mirror-2:19.1.1-373.g4e45e6b4.el9.x86_64 36/61 2024-09-21T09:42:59.293 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: rbd-mirror-2:19.1.1-373.g4e45e6b4.el9.x86_64 36/61 2024-09-21T09:42:59.298 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-mgr-2:19.1.1-373.g4e45e6b4.el9.x86_64 37/61 2024-09-21T09:42:59.299 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : ceph-mgr-2:19.1.1-373.g4e45e6b4.el9.x86_64 37/61 2024-09-21T09:42:59.327 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-mgr-2:19.1.1-373.g4e45e6b4.el9.x86_64 37/61 2024-09-21T09:42:59.352 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : rbd-fuse-2:19.1.1-373.g4e45e6b4.el9.x86_64 38/61 2024-09-21T09:42:59.357 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.1.1-373.g4e45e6b4 39/61 2024-09-21T09:42:59.357 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : ceph-immutable-object-cache-2:19.1.1-373.g4e45e6b4 39/61 2024-09-21T09:42:59.386 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.1.1-373.g4e45e6b4 39/61 2024-09-21T09:42:59.403 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : libcephfs-devel-2:19.1.1-373.g4e45e6b4.el9.x86_64 40/61 2024-09-21T09:42:59.403 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : libcephsqlite-2:19.1.1-373.g4e45e6b4.el9.x86_64 41/61 2024-09-21T09:42:59.479 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: libcephsqlite-2:19.1.1-373.g4e45e6b4.el9.x86_64 41/61 2024-09-21T09:42:59.512 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : librados-devel-2:19.1.1-373.g4e45e6b4.el9.x86_64 42/61 2024-09-21T09:42:59.519 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-mds-2:19.1.1-373.g4e45e6b4.el9.x86_64 43/61 2024-09-21T09:42:59.519 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : ceph-mds-2:19.1.1-373.g4e45e6b4.el9.x86_64 43/61 2024-09-21T09:42:59.555 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-mds-2:19.1.1-373.g4e45e6b4.el9.x86_64 43/61 2024-09-21T09:42:59.560 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-mon-2:19.1.1-373.g4e45e6b4.el9.x86_64 44/61 2024-09-21T09:42:59.560 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : ceph-mon-2:19.1.1-373.g4e45e6b4.el9.x86_64 44/61 2024-09-21T09:42:59.596 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-mon-2:19.1.1-373.g4e45e6b4.el9.x86_64 44/61 2024-09-21T09:42:59.602 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-osd-2:19.1.1-373.g4e45e6b4.el9.x86_64 45/61 2024-09-21T09:42:59.602 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : ceph-osd-2:19.1.1-373.g4e45e6b4.el9.x86_64 45/61 2024-09-21T09:42:59.638 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-osd-2:19.1.1-373.g4e45e6b4.el9.x86_64 45/61 2024-09-21T09:42:59.639 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : ceph-selinux-2:19.1.1-373.g4e45e6b4.el9.x86_64 46/61 2024-09-21T09:42:59.672 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-selinux-2:19.1.1-373.g4e45e6b4.el9.x86_64 46/61 2024-09-21T09:42:59.679 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-base-2:19.1.1-373.g4e45e6b4.el9.x86_64 47/61 2024-09-21T09:42:59.679 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : ceph-base-2:19.1.1-373.g4e45e6b4.el9.x86_64 47/61 2024-09-21T09:42:59.784 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-base-2:19.1.1-373.g4e45e6b4.el9.x86_64 47/61 2024-09-21T09:42:59.784 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : ceph-common-2:19.1.1-373.g4e45e6b4.el9.x86_64 48/61 2024-09-21T09:42:59.818 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: ceph-common-2:19.1.1-373.g4e45e6b4.el9.x86_64 48/61 2024-09-21T09:42:59.818 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : libradosstriper1-2:19.1.1-373.g4e45e6b4.el9.x86_64 49/61 2024-09-21T09:42:59.902 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: libradosstriper1-2:19.1.1-373.g4e45e6b4.el9.x86_64 49/61 2024-09-21T09:42:59.918 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : python3-cephfs-2:19.1.1-373.g4e45e6b4.el9.x86_64 50/61 2024-09-21T09:42:59.944 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : python3-rbd-2:19.1.1-373.g4e45e6b4.el9.x86_64 51/61 2024-09-21T09:42:59.944 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : librbd1-2:19.1.1-373.g4e45e6b4.el9.x86_64 52/61 2024-09-21T09:43:00.020 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: librbd1-2:19.1.1-373.g4e45e6b4.el9.x86_64 52/61 2024-09-21T09:43:00.045 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : python3-rgw-2:19.1.1-373.g4e45e6b4.el9.x86_64 53/61 2024-09-21T09:43:00.045 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : librgw2-2:19.1.1-373.g4e45e6b4.el9.x86_64 54/61 2024-09-21T09:43:00.130 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: librgw2-2:19.1.1-373.g4e45e6b4.el9.x86_64 54/61 2024-09-21T09:43:00.154 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : python3-rados-2:19.1.1-373.g4e45e6b4.el9.x86_64 55/61 2024-09-21T09:43:00.155 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : libcephfs2-2:19.1.1-373.g4e45e6b4.el9.x86_64 56/61 2024-09-21T09:43:00.239 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: libcephfs2-2:19.1.1-373.g4e45e6b4.el9.x86_64 56/61 2024-09-21T09:43:00.264 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : ceph-fuse-2:19.1.1-373.g4e45e6b4.el9.x86_64 57/61 2024-09-21T09:43:00.289 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : python3-ceph-argparse-2:19.1.1-373.g4e45e6b4.el9.x 58/61 2024-09-21T09:43:00.314 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : python3-ceph-common-2:19.1.1-373.g4e45e6b4.el9.x86 59/61 2024-09-21T09:43:00.340 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : ceph-mgr-modules-core-2:19.1.1-373.g4e45e6b4.el9.n 60/61 2024-09-21T09:43:00.340 INFO:teuthology.orchestra.run.smithi092.stdout: Cleanup : librados2-2:19.1.1-373.g4e45e6b4.el9.x86_64 61/61 2024-09-21T09:43:01.431 INFO:teuthology.orchestra.run.smithi092.stdout: Running scriptlet: librados2-2:19.1.1-373.g4e45e6b4.el9.x86_64 61/61 2024-09-21T09:43:01.431 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : libnbd-1.20.2-2.el9.x86_64 1/61 2024-09-21T09:43:01.432 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-2:19.3.0-5092.g62da54b4.el9.x86_64 2/61 2024-09-21T09:43:01.432 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-2:19.1.1-373.g4e45e6b4.el9.x86_64 3/61 2024-09-21T09:43:01.432 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-base-2:19.3.0-5092.g62da54b4.el9.x86_64 4/61 2024-09-21T09:43:01.432 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-base-2:19.1.1-373.g4e45e6b4.el9.x86_64 5/61 2024-09-21T09:43:01.432 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-common-2:19.3.0-5092.g62da54b4.el9.x86_64 6/61 2024-09-21T09:43:01.432 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-common-2:19.1.1-373.g4e45e6b4.el9.x86_64 7/61 2024-09-21T09:43:01.432 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-fuse-2:19.3.0-5092.g62da54b4.el9.x86_64 8/61 2024-09-21T09:43:01.432 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-fuse-2:19.1.1-373.g4e45e6b4.el9.x86_64 9/61 2024-09-21T09:43:01.432 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-immutable-object-cache-2:19.3.0-5092.g62da54b 10/61 2024-09-21T09:43:01.432 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-immutable-object-cache-2:19.1.1-373.g4e45e6b4 11/61 2024-09-21T09:43:01.433 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-mds-2:19.3.0-5092.g62da54b4.el9.x86_64 12/61 2024-09-21T09:43:01.433 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-mds-2:19.1.1-373.g4e45e6b4.el9.x86_64 13/61 2024-09-21T09:43:01.433 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-mgr-2:19.3.0-5092.g62da54b4.el9.x86_64 14/61 2024-09-21T09:43:01.433 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-mgr-2:19.1.1-373.g4e45e6b4.el9.x86_64 15/61 2024-09-21T09:43:01.433 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-mon-2:19.3.0-5092.g62da54b4.el9.x86_64 16/61 2024-09-21T09:43:01.433 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-mon-2:19.1.1-373.g4e45e6b4.el9.x86_64 17/61 2024-09-21T09:43:01.433 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-osd-2:19.3.0-5092.g62da54b4.el9.x86_64 18/61 2024-09-21T09:43:01.433 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-osd-2:19.1.1-373.g4e45e6b4.el9.x86_64 19/61 2024-09-21T09:43:01.433 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-radosgw-2:19.3.0-5092.g62da54b4.el9.x86_64 20/61 2024-09-21T09:43:01.433 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-radosgw-2:19.1.1-373.g4e45e6b4.el9.x86_64 21/61 2024-09-21T09:43:01.433 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-selinux-2:19.3.0-5092.g62da54b4.el9.x86_64 22/61 2024-09-21T09:43:01.433 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-selinux-2:19.1.1-373.g4e45e6b4.el9.x86_64 23/61 2024-09-21T09:43:01.433 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-test-2:19.3.0-5092.g62da54b4.el9.x86_64 24/61 2024-09-21T09:43:01.433 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-test-2:19.1.1-373.g4e45e6b4.el9.x86_64 25/61 2024-09-21T09:43:01.434 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : libcephfs-devel-2:19.3.0-5092.g62da54b4.el9.x86_64 26/61 2024-09-21T09:43:01.434 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : libcephfs-devel-2:19.1.1-373.g4e45e6b4.el9.x86_64 27/61 2024-09-21T09:43:01.434 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : libcephfs2-2:19.3.0-5092.g62da54b4.el9.x86_64 28/61 2024-09-21T09:43:01.434 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : libcephfs2-2:19.1.1-373.g4e45e6b4.el9.x86_64 29/61 2024-09-21T09:43:01.434 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : libcephsqlite-2:19.3.0-5092.g62da54b4.el9.x86_64 30/61 2024-09-21T09:43:01.437 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : libcephsqlite-2:19.1.1-373.g4e45e6b4.el9.x86_64 31/61 2024-09-21T09:43:01.437 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : librados-devel-2:19.3.0-5092.g62da54b4.el9.x86_64 32/61 2024-09-21T09:43:01.437 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : librados-devel-2:19.1.1-373.g4e45e6b4.el9.x86_64 33/61 2024-09-21T09:43:01.437 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : librados2-2:19.3.0-5092.g62da54b4.el9.x86_64 34/61 2024-09-21T09:43:01.437 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : librados2-2:19.1.1-373.g4e45e6b4.el9.x86_64 35/61 2024-09-21T09:43:01.437 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : libradosstriper1-2:19.3.0-5092.g62da54b4.el9.x86_6 36/61 2024-09-21T09:43:01.437 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : libradosstriper1-2:19.1.1-373.g4e45e6b4.el9.x86_64 37/61 2024-09-21T09:43:01.437 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : librbd1-2:19.3.0-5092.g62da54b4.el9.x86_64 38/61 2024-09-21T09:43:01.437 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : librbd1-2:19.1.1-373.g4e45e6b4.el9.x86_64 39/61 2024-09-21T09:43:01.437 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : librgw2-2:19.3.0-5092.g62da54b4.el9.x86_64 40/61 2024-09-21T09:43:01.437 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : librgw2-2:19.1.1-373.g4e45e6b4.el9.x86_64 41/61 2024-09-21T09:43:01.437 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-ceph-argparse-2:19.3.0-5092.g62da54b4.el9. 42/61 2024-09-21T09:43:01.437 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-ceph-argparse-2:19.1.1-373.g4e45e6b4.el9.x 43/61 2024-09-21T09:43:01.438 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-ceph-common-2:19.3.0-5092.g62da54b4.el9.x8 44/61 2024-09-21T09:43:01.438 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-ceph-common-2:19.1.1-373.g4e45e6b4.el9.x86 45/61 2024-09-21T09:43:01.438 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-cephfs-2:19.3.0-5092.g62da54b4.el9.x86_64 46/61 2024-09-21T09:43:01.438 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-cephfs-2:19.1.1-373.g4e45e6b4.el9.x86_64 47/61 2024-09-21T09:43:01.438 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-rados-2:19.3.0-5092.g62da54b4.el9.x86_64 48/61 2024-09-21T09:43:01.438 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-rados-2:19.1.1-373.g4e45e6b4.el9.x86_64 49/61 2024-09-21T09:43:01.438 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-rbd-2:19.3.0-5092.g62da54b4.el9.x86_64 50/61 2024-09-21T09:43:01.438 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-rbd-2:19.1.1-373.g4e45e6b4.el9.x86_64 51/61 2024-09-21T09:43:01.438 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-rgw-2:19.3.0-5092.g62da54b4.el9.x86_64 52/61 2024-09-21T09:43:01.438 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : python3-rgw-2:19.1.1-373.g4e45e6b4.el9.x86_64 53/61 2024-09-21T09:43:01.438 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : rbd-fuse-2:19.3.0-5092.g62da54b4.el9.x86_64 54/61 2024-09-21T09:43:01.438 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : rbd-fuse-2:19.1.1-373.g4e45e6b4.el9.x86_64 55/61 2024-09-21T09:43:01.438 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : rbd-mirror-2:19.3.0-5092.g62da54b4.el9.x86_64 56/61 2024-09-21T09:43:01.438 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : rbd-mirror-2:19.1.1-373.g4e45e6b4.el9.x86_64 57/61 2024-09-21T09:43:01.439 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : rbd-nbd-2:19.3.0-5092.g62da54b4.el9.x86_64 58/61 2024-09-21T09:43:01.439 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : rbd-nbd-2:19.1.1-373.g4e45e6b4.el9.x86_64 59/61 2024-09-21T09:43:01.439 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-mgr-modules-core-2:19.3.0-5092.g62da54b4.el9. 60/61 2024-09-21T09:43:02.636 INFO:teuthology.orchestra.run.smithi092.stdout: Verifying : ceph-mgr-modules-core-2:19.1.1-373.g4e45e6b4.el9.n 61/61 2024-09-21T09:43:02.636 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:02.636 INFO:teuthology.orchestra.run.smithi092.stdout:Upgraded: 2024-09-21T09:43:02.636 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.636 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-base-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.636 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-common-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.636 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-fuse-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.636 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-immutable-object-cache-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.636 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mds-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.636 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mgr-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.637 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mgr-modules-core-2:19.3.0-5092.g62da54b4.el9.noarch 2024-09-21T09:43:02.637 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-mon-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.637 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-osd-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.637 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-radosgw-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.637 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-selinux-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.637 INFO:teuthology.orchestra.run.smithi092.stdout: ceph-test-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.637 INFO:teuthology.orchestra.run.smithi092.stdout: libcephfs-devel-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.637 INFO:teuthology.orchestra.run.smithi092.stdout: libcephfs2-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.637 INFO:teuthology.orchestra.run.smithi092.stdout: libcephsqlite-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.637 INFO:teuthology.orchestra.run.smithi092.stdout: librados-devel-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.637 INFO:teuthology.orchestra.run.smithi092.stdout: librados2-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.637 INFO:teuthology.orchestra.run.smithi092.stdout: libradosstriper1-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.637 INFO:teuthology.orchestra.run.smithi092.stdout: librbd1-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.637 INFO:teuthology.orchestra.run.smithi092.stdout: librgw2-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.638 INFO:teuthology.orchestra.run.smithi092.stdout: python3-ceph-argparse-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.638 INFO:teuthology.orchestra.run.smithi092.stdout: python3-ceph-common-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.638 INFO:teuthology.orchestra.run.smithi092.stdout: python3-cephfs-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.638 INFO:teuthology.orchestra.run.smithi092.stdout: python3-rados-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.638 INFO:teuthology.orchestra.run.smithi092.stdout: python3-rbd-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.638 INFO:teuthology.orchestra.run.smithi092.stdout: python3-rgw-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.638 INFO:teuthology.orchestra.run.smithi092.stdout: rbd-fuse-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.638 INFO:teuthology.orchestra.run.smithi092.stdout: rbd-mirror-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.638 INFO:teuthology.orchestra.run.smithi092.stdout: rbd-nbd-2:19.3.0-5092.g62da54b4.el9.x86_64 2024-09-21T09:43:02.638 INFO:teuthology.orchestra.run.smithi092.stdout:Installed: 2024-09-21T09:43:02.638 INFO:teuthology.orchestra.run.smithi092.stdout: libnbd-1.20.2-2.el9.x86_64 2024-09-21T09:43:02.638 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:02.639 INFO:teuthology.orchestra.run.smithi092.stdout:Complete! 2024-09-21T09:43:02.982 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:43:03.124 DEBUG:teuthology.orchestra.run.smithi092:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-21T09:43:03.147 INFO:teuthology.orchestra.run.smithi092.stdout:19.3.0-5092.g62da54b4.el9 2024-09-21T09:43:03.147 INFO:teuthology.packaging:The installed version of ceph is 19.3.0-5092.g62da54b4.el9 2024-09-21T09:43:03.147 INFO:teuthology.task.install:The correct ceph version 19.3.0-5092.g62da54b4 is installed. 2024-09-21T09:43:03.149 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:43:03.159 INFO:teuthology.task.print:**** done install.upgrade 2024-09-21T09:43:03.159 INFO:teuthology.run_tasks:Running task ceph.restart... 2024-09-21T09:43:03.207 DEBUG:tasks.ceph.mon.a:waiting for process to exit 2024-09-21T09:43:03.207 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:43:03.219 INFO:tasks.ceph.mon.a:Stopped 2024-09-21T09:43:03.219 INFO:tasks.ceph.mon.a:Restarting daemon 2024-09-21T09:43:03.219 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i a 2024-09-21T09:43:03.261 INFO:tasks.ceph.mon.a:Started 2024-09-21T09:43:03.262 DEBUG:tasks.ceph.mon.b:waiting for process to exit 2024-09-21T09:43:03.262 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:43:03.274 INFO:tasks.ceph.mon.b:Stopped 2024-09-21T09:43:03.274 INFO:tasks.ceph.mon.b:Restarting daemon 2024-09-21T09:43:03.275 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i b 2024-09-21T09:43:03.317 INFO:tasks.ceph.mon.b:Started 2024-09-21T09:43:03.317 DEBUG:tasks.ceph.mon.c:waiting for process to exit 2024-09-21T09:43:03.317 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:43:03.330 INFO:tasks.ceph.mon.c:Stopped 2024-09-21T09:43:03.330 INFO:tasks.ceph.mon.c:Restarting daemon 2024-09-21T09:43:03.330 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i c 2024-09-21T09:43:03.373 INFO:tasks.ceph.mon.c:Started 2024-09-21T09:43:03.373 DEBUG:tasks.ceph.mgr.x:waiting for process to exit 2024-09-21T09:43:03.373 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:43:03.393 INFO:tasks.ceph.mgr.x:Stopped 2024-09-21T09:43:03.393 INFO:tasks.ceph.mgr.x:Restarting daemon 2024-09-21T09:43:03.393 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i x 2024-09-21T09:43:03.436 INFO:tasks.ceph.mgr.x:Started 2024-09-21T09:43:03.436 DEBUG:tasks.ceph.mgr.y:waiting for process to exit 2024-09-21T09:43:03.436 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:43:03.460 INFO:tasks.ceph.mgr.y:Stopped 2024-09-21T09:43:03.460 INFO:tasks.ceph.mgr.y:Restarting daemon 2024-09-21T09:43:03.461 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i y 2024-09-21T09:43:03.503 INFO:tasks.ceph.mgr.y:Started 2024-09-21T09:43:03.503 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-21T09:43:03.512 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T09:43:03.512 INFO:tasks.ceph.ceph_manager.ceph:waiting for mgr available 2024-09-21T09:43:03.512 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mgr dump --format=json 2024-09-21T09:43:03.622 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.620+0000 7fca2c1da640 1 -- 172.21.15.92:0/1459861316 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fca240916e0 msgr2=0x7fca24134d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:03.622 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.620+0000 7fca2c1da640 1 --2- 172.21.15.92:0/1459861316 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fca240916e0 0x7fca24134d80 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7fca1c005fd0 tx=0x7fca1c02a780 comp rx=0 tx=0).stop 2024-09-21T09:43:03.622 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.620+0000 7fca2c1da640 1 -- 172.21.15.92:0/1459861316 shutdown_connections 2024-09-21T09:43:03.622 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.620+0000 7fca2c1da640 1 --2- 172.21.15.92:0/1459861316 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fca240916e0 0x7fca24134d80 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:03.622 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.620+0000 7fca2c1da640 1 --2- 172.21.15.92:0/1459861316 >> v2:172.21.15.92:3300/0 conn(0x7fca2406e2d0 0x7fca2406e6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:03.623 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.620+0000 7fca2c1da640 1 -- 172.21.15.92:0/1459861316 >> 172.21.15.92:0/1459861316 conn(0x7fca2409c150 msgr2=0x7fca2409c560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:03.623 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.620+0000 7fca2c1da640 1 -- 172.21.15.92:0/1459861316 shutdown_connections 2024-09-21T09:43:03.623 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.620+0000 7fca2c1da640 1 -- 172.21.15.92:0/1459861316 wait complete. 2024-09-21T09:43:03.624 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.621+0000 7fca2c1da640 1 Processor -- start 2024-09-21T09:43:03.624 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.621+0000 7fca2c1da640 1 -- start start 2024-09-21T09:43:03.624 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.621+0000 7fca2c1da640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fca2406e2d0 0x7fca2406f7e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:03.625 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.621+0000 7fca2c1da640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fca2406ebf0 0x7fca2406fd20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:03.625 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.621+0000 7fca2c1da640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fca240916e0 0x7fca24070260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:03.625 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.621+0000 7fca2c1da640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fca24135760 con 0x7fca2406ebf0 2024-09-21T09:43:03.625 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.621+0000 7fca2c1da640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fca241355e0 con 0x7fca240916e0 2024-09-21T09:43:03.625 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.621+0000 7fca2c1da640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fca241358e0 con 0x7fca2406e2d0 2024-09-21T09:43:03.625 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.622+0000 7fca29775640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fca2406ebf0 0x7fca2406fd20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:03.625 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.622+0000 7fca2a777640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fca240916e0 0x7fca24070260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:03.625 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.622+0000 7fca29775640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fca2406ebf0 0x7fca2406fd20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:48204/0 (socket says 172.21.15.92:48204) 2024-09-21T09:43:03.625 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.622+0000 7fca29775640 1 -- 172.21.15.92:0/3799820380 learned_addr learned my addr 172.21.15.92:0/3799820380 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:03.625 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.622+0000 7fca2a777640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fca240916e0 0x7fca24070260 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:34648/0 (socket says 172.21.15.92:34648) 2024-09-21T09:43:03.625 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.622+0000 7fca29f76640 1 --2- 172.21.15.92:0/3799820380 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fca2406e2d0 0x7fca2406f7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:03.625 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.622+0000 7fca29775640 1 -- 172.21.15.92:0/3799820380 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fca2406e2d0 msgr2=0x7fca2406f7e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:03.625 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.622+0000 7fca29775640 1 --2- 172.21.15.92:0/3799820380 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fca2406e2d0 0x7fca2406f7e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:03.625 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.622+0000 7fca29775640 1 -- 172.21.15.92:0/3799820380 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fca240916e0 msgr2=0x7fca24070260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:03.625 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.622+0000 7fca29775640 1 --2- 172.21.15.92:0/3799820380 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fca240916e0 0x7fca24070260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:03.626 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.622+0000 7fca29775640 1 -- 172.21.15.92:0/3799820380 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fca24070e90 con 0x7fca2406ebf0 2024-09-21T09:43:03.626 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.622+0000 7fca29775640 1 --2- 172.21.15.92:0/3799820380 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fca2406ebf0 0x7fca2406fd20 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7fca1400c500 tx=0x7fca1400c900 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:03.626 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.622+0000 7fca1a7fc640 1 -- 172.21.15.92:0/3799820380 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fca14017070 con 0x7fca2406ebf0 2024-09-21T09:43:03.626 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.622+0000 7fca1a7fc640 1 -- 172.21.15.92:0/3799820380 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fca140053c0 con 0x7fca2406ebf0 2024-09-21T09:43:03.626 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.622+0000 7fca2c1da640 1 -- 172.21.15.92:0/3799820380 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fca24071320 con 0x7fca2406ebf0 2024-09-21T09:43:03.626 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.622+0000 7fca1a7fc640 1 -- 172.21.15.92:0/3799820380 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fca14003850 con 0x7fca2406ebf0 2024-09-21T09:43:03.626 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.622+0000 7fca2c1da640 1 -- 172.21.15.92:0/3799820380 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fca24071970 con 0x7fca2406ebf0 2024-09-21T09:43:03.627 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.623+0000 7fca1a7fc640 1 -- 172.21.15.92:0/3799820380 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fca1401a070 con 0x7fca2406ebf0 2024-09-21T09:43:03.627 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.623+0000 7fca2c1da640 1 -- 172.21.15.92:0/3799820380 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fca2406e6b0 con 0x7fca2406ebf0 2024-09-21T09:43:03.627 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.624+0000 7fca29f76640 1 -- 172.21.15.92:0/3799820380 >> v1:172.21.15.92:6800/1280404317 conn(0x7fca08047fd0 legacy=0x7fca0804a490 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.92:6800/1280404317 2024-09-21T09:43:03.627 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.624+0000 7fca1a7fc640 1 -- 172.21.15.92:0/3799820380 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7fca1405ce40 con 0x7fca2406ebf0 2024-09-21T09:43:03.629 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.627+0000 7fca1a7fc640 1 -- 172.21.15.92:0/3799820380 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+152684 (secure 0 0 0) 0x7fca1405bfd0 con 0x7fca2406ebf0 2024-09-21T09:43:03.808 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.806+0000 7fca2c1da640 1 -- 172.21.15.92:0/3799820380 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "mgr dump", "format": "json"} v 0) -- 0x7fca241c8b00 con 0x7fca2406ebf0 2024-09-21T09:43:03.811 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.809+0000 7fca1a7fc640 1 -- 172.21.15.92:0/3799820380 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json"}]=0 v5) ==== 74+0+108899 (secure 0 0 0) 0x7fca1403c0c0 con 0x7fca2406ebf0 2024-09-21T09:43:03.812 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:03.814 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.811+0000 7fca2c1da640 1 -- 172.21.15.92:0/3799820380 >> v1:172.21.15.92:6800/1280404317 conn(0x7fca08047fd0 legacy=0x7fca0804a490 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:43:03.814 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.811+0000 7fca2c1da640 1 -- 172.21.15.92:0/3799820380 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fca2406ebf0 msgr2=0x7fca2406fd20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:03.814 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.811+0000 7fca2c1da640 1 --2- 172.21.15.92:0/3799820380 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fca2406ebf0 0x7fca2406fd20 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7fca1400c500 tx=0x7fca1400c900 comp rx=0 tx=0).stop 2024-09-21T09:43:03.814 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.812+0000 7fca2c1da640 1 -- 172.21.15.92:0/3799820380 shutdown_connections 2024-09-21T09:43:03.814 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.812+0000 7fca2c1da640 1 --2- 172.21.15.92:0/3799820380 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fca240916e0 0x7fca24070260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:03.814 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.812+0000 7fca2c1da640 1 --2- 172.21.15.92:0/3799820380 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fca2406ebf0 0x7fca2406fd20 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:03.814 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.812+0000 7fca2c1da640 1 --2- 172.21.15.92:0/3799820380 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fca2406e2d0 0x7fca2406f7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:03.814 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.812+0000 7fca2c1da640 1 -- 172.21.15.92:0/3799820380 >> 172.21.15.92:0/3799820380 conn(0x7fca2409c150 msgr2=0x7fca24098a70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:03.814 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.812+0000 7fca2c1da640 1 -- 172.21.15.92:0/3799820380 shutdown_connections 2024-09-21T09:43:03.814 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.812+0000 7fca2c1da640 1 -- 172.21.15.92:0/3799820380 wait complete. 2024-09-21T09:43:03.825 INFO:teuthology.orchestra.run.smithi092.stdout:{"epoch":5,"flags":0,"active_gid":4146,"active_name":"y","active_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6800","nonce":1280404317}]},"active_addr":"172.21.15.92:6800/1280404317","active_change":"2024-09-21T09:41:38.005482+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":4120,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.92:0","nonce":3587832392}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.92:0","nonce":230938560}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.92:0","nonce":2147030555}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.92:0","nonce":162460762}]}]} 2024-09-21T09:43:03.827 INFO:tasks.ceph.ceph_manager.ceph:mgr available! 2024-09-21T09:43:03.827 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up 2024-09-21T09:43:03.828 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:43:03.929 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.927+0000 7f2fe8908640 1 -- 172.21.15.92:0/505867429 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2fe41346f0 msgr2=0x7f2fe4136ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:03.929 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.927+0000 7f2fe8908640 1 --2- 172.21.15.92:0/505867429 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2fe41346f0 0x7f2fe4136ae0 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f2fdc005fd0 tx=0x7f2fdc02aa10 comp rx=0 tx=0).stop 2024-09-21T09:43:03.930 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.927+0000 7f2fe8908640 1 -- 172.21.15.92:0/505867429 shutdown_connections 2024-09-21T09:43:03.930 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.927+0000 7f2fe8908640 1 --2- 172.21.15.92:0/505867429 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2fe41346f0 0x7f2fe4136ae0 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:03.930 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.927+0000 7f2fe8908640 1 --2- 172.21.15.92:0/505867429 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2fe4131dc0 0x7f2fe41341b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:03.930 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.928+0000 7f2fe8908640 1 -- 172.21.15.92:0/505867429 >> 172.21.15.92:0/505867429 conn(0x7f2fe40a1800 msgr2=0x7f2fe40a1c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:03.930 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.928+0000 7f2fe8908640 1 -- 172.21.15.92:0/505867429 shutdown_connections 2024-09-21T09:43:03.930 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.928+0000 7f2fe8908640 1 -- 172.21.15.92:0/505867429 wait complete. 2024-09-21T09:43:03.932 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.929+0000 7f2fe8908640 1 Processor -- start 2024-09-21T09:43:03.932 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.929+0000 7f2fe8908640 1 -- start start 2024-09-21T09:43:03.932 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.929+0000 7f2fe8908640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2fe4099080 0x7f2fe412c920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:03.932 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.929+0000 7f2fe8908640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2fe4131dc0 0x7f2fe412ce60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:03.932 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.929+0000 7f2fe8908640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2fe41346f0 0x7f2fe412d3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:03.932 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.929+0000 7f2fe8908640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f2fe41378d0 con 0x7f2fe4131dc0 2024-09-21T09:43:03.932 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.929+0000 7f2fe8908640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f2fe4137750 con 0x7f2fe4099080 2024-09-21T09:43:03.933 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.929+0000 7f2fe8908640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f2fe4137a50 con 0x7f2fe41346f0 2024-09-21T09:43:03.933 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.929+0000 7f2fdbfff640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2fe4131dc0 0x7f2fe412ce60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:03.933 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.929+0000 7f2fdbfff640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2fe4131dc0 0x7f2fe412ce60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:48250/0 (socket says 172.21.15.92:48250) 2024-09-21T09:43:03.933 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.929+0000 7f2fdbfff640 1 -- 172.21.15.92:0/2981310407 learned_addr learned my addr 172.21.15.92:0/2981310407 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:03.933 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.929+0000 7f2fe259c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2fe4099080 0x7f2fe412c920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:03.933 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.929+0000 7f2fe2d9d640 1 --2- 172.21.15.92:0/2981310407 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2fe41346f0 0x7f2fe412d3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:03.933 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.929+0000 7f2fdbfff640 1 -- 172.21.15.92:0/2981310407 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2fe41346f0 msgr2=0x7f2fe412d3a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:03.933 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.930+0000 7f2fdbfff640 1 --2- 172.21.15.92:0/2981310407 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2fe41346f0 0x7f2fe412d3a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:03.933 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.930+0000 7f2fdbfff640 1 -- 172.21.15.92:0/2981310407 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2fe4099080 msgr2=0x7f2fe412c920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:03.933 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.930+0000 7f2fdbfff640 1 --2- 172.21.15.92:0/2981310407 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2fe4099080 0x7f2fe412c920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:03.933 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.930+0000 7f2fdbfff640 1 -- 172.21.15.92:0/2981310407 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2fe409df10 con 0x7f2fe4131dc0 2024-09-21T09:43:03.933 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.930+0000 7f2fe259c640 1 --2- 172.21.15.92:0/2981310407 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2fe4099080 0x7f2fe412c920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:03.933 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.930+0000 7f2fe2d9d640 1 --2- 172.21.15.92:0/2981310407 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2fe41346f0 0x7f2fe412d3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:03.933 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.930+0000 7f2fdbfff640 1 --2- 172.21.15.92:0/2981310407 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2fe4131dc0 0x7f2fe412ce60 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f2fcc00ea60 tx=0x7f2fcc00ef20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:03.934 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.930+0000 7f2fdaffd640 1 -- 172.21.15.92:0/2981310407 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2fcc00cde0 con 0x7f2fe4131dc0 2024-09-21T09:43:03.934 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.930+0000 7f2fdaffd640 1 -- 172.21.15.92:0/2981310407 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2fcc004020 con 0x7f2fe4131dc0 2024-09-21T09:43:03.934 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.930+0000 7f2fe8908640 1 -- 172.21.15.92:0/2981310407 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2fe41294d0 con 0x7f2fe4131dc0 2024-09-21T09:43:03.934 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.930+0000 7f2fdaffd640 1 -- 172.21.15.92:0/2981310407 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2fcc004450 con 0x7f2fe4131dc0 2024-09-21T09:43:03.934 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.930+0000 7f2fe8908640 1 -- 172.21.15.92:0/2981310407 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2fe41cad10 con 0x7f2fe4131dc0 2024-09-21T09:43:03.934 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.931+0000 7f2fe8908640 1 -- 172.21.15.92:0/2981310407 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2fe4132910 con 0x7f2fe4131dc0 2024-09-21T09:43:03.934 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.931+0000 7f2fdaffd640 1 -- 172.21.15.92:0/2981310407 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f2fcc0035f0 con 0x7f2fe4131dc0 2024-09-21T09:43:03.935 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.932+0000 7f2fe259c640 1 -- 172.21.15.92:0/2981310407 >> v1:172.21.15.92:6800/1280404317 conn(0x7f2fc4047fd0 legacy=0x7f2fc404a490 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.92:6800/1280404317 2024-09-21T09:43:03.935 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.932+0000 7f2fdaffd640 1 -- 172.21.15.92:0/2981310407 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7f2fcc05e1f0 con 0x7f2fe4131dc0 2024-09-21T09:43:03.937 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:03.935+0000 7f2fdaffd640 1 -- 172.21.15.92:0/2981310407 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+152684 (secure 0 0 0) 0x7f2fe4132910 con 0x7f2fe4131dc0 2024-09-21T09:43:04.087 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.085+0000 7f2fe8908640 1 -- 172.21.15.92:0/2981310407 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f2fe4099460 con 0x7f2fe4131dc0 2024-09-21T09:43:04.087 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.086+0000 7f2fdaffd640 1 -- 172.21.15.92:0/2981310407 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) ==== 74+0+8812 (secure 0 0 0) 0x7f2fcc05d380 con 0x7f2fe4131dc0 2024-09-21T09:43:04.087 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:04.088 INFO:teuthology.orchestra.run.smithi092.stdout:{"epoch":20,"fsid":"5f250e6c-4e36-49b1-ab62-d9454369fa03","created":"2024-09-21T09:41:34.680338+0000","modified":"2024-09-21T09:42:01.457080+0000","last_up_change":"2024-09-21T09:41:48.412163+0000","last_in_change":"2024-09-21T09:41:37.598778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"squid","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:41:50.040313+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:41:52.054798+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f3314f59-3374-4072-85c9-f8c4e458efc8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6805","nonce":195102377}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6806","nonce":195102377}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6808","nonce":195102377}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6807","nonce":195102377}]},"public_addr":"172.21.15.92:6805/195102377","cluster_addr":"172.21.15.92:6806/195102377","heartbeat_back_addr":"172.21.15.92:6808/195102377","heartbeat_front_addr":"172.21.15.92:6807/195102377","state":["exists","up"]},{"osd":1,"uuid":"cf0486af-ba68-4763-a364-c1f2181604a7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6813","nonce":1359151798}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6814","nonce":1359151798}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6816","nonce":1359151798}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6815","nonce":1359151798}]},"public_addr":"172.21.15.92:6813/1359151798","cluster_addr":"172.21.15.92:6814/1359151798","heartbeat_back_addr":"172.21.15.92:6816/1359151798","heartbeat_front_addr":"172.21.15.92:6815/1359151798","state":["exists","up"]},{"osd":2,"uuid":"5c587421-3409-4469-a2ee-b2ea20cd45e5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6809","nonce":296401696}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6810","nonce":296401696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6812","nonce":296401696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6811","nonce":296401696}]},"public_addr":"172.21.15.92:6809/296401696","cluster_addr":"172.21.15.92:6810/296401696","heartbeat_back_addr":"172.21.15.92:6812/296401696","heartbeat_front_addr":"172.21.15.92:6811/296401696","state":["exists","up"]},{"osd":3,"uuid":"b98663b7-de4d-4c12-ac91-8c33f159da35","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6801","nonce":1829519295}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6802","nonce":1829519295}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6804","nonce":1829519295}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6803","nonce":1829519295}]},"public_addr":"172.21.15.92:6801/1829519295","cluster_addr":"172.21.15.92:6802/1829519295","heartbeat_back_addr":"172.21.15.92:6804/1829519295","heartbeat_front_addr":"172.21.15.92:6803/1829519295","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.934317+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:45.144020+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.895848+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.636219+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:43:04.089 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.088+0000 7f2fe8908640 1 -- 172.21.15.92:0/2981310407 >> v1:172.21.15.92:6800/1280404317 conn(0x7f2fc4047fd0 legacy=0x7f2fc404a490 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:43:04.090 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.088+0000 7f2fe8908640 1 -- 172.21.15.92:0/2981310407 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2fe4131dc0 msgr2=0x7f2fe412ce60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.090 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.088+0000 7f2fe8908640 1 --2- 172.21.15.92:0/2981310407 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2fe4131dc0 0x7f2fe412ce60 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f2fcc00ea60 tx=0x7f2fcc00ef20 comp rx=0 tx=0).stop 2024-09-21T09:43:04.090 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.088+0000 7f2fe8908640 1 -- 172.21.15.92:0/2981310407 shutdown_connections 2024-09-21T09:43:04.090 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.088+0000 7f2fe8908640 1 --2- 172.21.15.92:0/2981310407 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2fe41346f0 0x7f2fe412d3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.090 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.088+0000 7f2fe8908640 1 --2- 172.21.15.92:0/2981310407 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2fe4131dc0 0x7f2fe412ce60 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.090 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.088+0000 7f2fe8908640 1 --2- 172.21.15.92:0/2981310407 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2fe4099080 0x7f2fe412c920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.090 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.088+0000 7f2fe8908640 1 -- 172.21.15.92:0/2981310407 >> 172.21.15.92:0/2981310407 conn(0x7f2fe40a1800 msgr2=0x7f2fe4132d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:04.090 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.088+0000 7f2fe8908640 1 -- 172.21.15.92:0/2981310407 shutdown_connections 2024-09-21T09:43:04.090 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.088+0000 7f2fe8908640 1 -- 172.21.15.92:0/2981310407 wait complete. 2024-09-21T09:43:04.101 INFO:tasks.ceph.ceph_manager.ceph:all up! 2024-09-21T09:43:04.101 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:43:04.202 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.200+0000 7f4f8bfd5640 1 -- 172.21.15.92:0/1022851835 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4f84131dc0 msgr2=0x7f4f841341b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.202 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.200+0000 7f4f8bfd5640 1 --2- 172.21.15.92:0/1022851835 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4f84131dc0 0x7f4f841341b0 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f4f74005fd0 tx=0x7f4f7402a540 comp rx=0 tx=0).stop 2024-09-21T09:43:04.202 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.200+0000 7f4f8bfd5640 1 -- 172.21.15.92:0/1022851835 shutdown_connections 2024-09-21T09:43:04.203 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.200+0000 7f4f8bfd5640 1 --2- 172.21.15.92:0/1022851835 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4f841346f0 0x7f4f84136ae0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.203 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.200+0000 7f4f8bfd5640 1 --2- 172.21.15.92:0/1022851835 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4f84131dc0 0x7f4f841341b0 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.203 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.200+0000 7f4f8bfd5640 1 -- 172.21.15.92:0/1022851835 >> 172.21.15.92:0/1022851835 conn(0x7f4f840a1800 msgr2=0x7f4f840a1c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:04.203 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.200+0000 7f4f8bfd5640 1 -- 172.21.15.92:0/1022851835 shutdown_connections 2024-09-21T09:43:04.203 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.200+0000 7f4f8bfd5640 1 -- 172.21.15.92:0/1022851835 wait complete. 2024-09-21T09:43:04.205 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.201+0000 7f4f8bfd5640 1 Processor -- start 2024-09-21T09:43:04.205 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.201+0000 7f4f8bfd5640 1 -- start start 2024-09-21T09:43:04.205 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.201+0000 7f4f8bfd5640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4f84099080 0x7f4f8409bde0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.205 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.201+0000 7f4f8bfd5640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4f84131dc0 0x7f4f8409c320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.205 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.201+0000 7f4f8bfd5640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4f841346f0 0x7f4f8409c860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.205 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.201+0000 7f4f8bfd5640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f4f84137970 con 0x7f4f84099080 2024-09-21T09:43:04.205 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.201+0000 7f4f8bfd5640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f4f841377f0 con 0x7f4f84131dc0 2024-09-21T09:43:04.205 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.201+0000 7f4f8bfd5640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f4f84137af0 con 0x7f4f841346f0 2024-09-21T09:43:04.205 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.202+0000 7f4f89d71640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4f84099080 0x7f4f8409bde0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.205 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.202+0000 7f4f89570640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4f84131dc0 0x7f4f8409c320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.205 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.202+0000 7f4f8a572640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4f841346f0 0x7f4f8409c860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.205 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.202+0000 7f4f89d71640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4f84099080 0x7f4f8409bde0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:48270/0 (socket says 172.21.15.92:48270) 2024-09-21T09:43:04.206 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.202+0000 7f4f89d71640 1 -- 172.21.15.92:0/163509414 learned_addr learned my addr 172.21.15.92:0/163509414 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:04.206 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.202+0000 7f4f89d71640 1 -- 172.21.15.92:0/163509414 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4f841346f0 msgr2=0x7f4f8409c860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.206 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.202+0000 7f4f89d71640 1 --2- 172.21.15.92:0/163509414 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4f841346f0 0x7f4f8409c860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.206 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.202+0000 7f4f89d71640 1 -- 172.21.15.92:0/163509414 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4f84131dc0 msgr2=0x7f4f8409c320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.206 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.202+0000 7f4f89d71640 1 --2- 172.21.15.92:0/163509414 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4f84131dc0 0x7f4f8409c320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.206 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.202+0000 7f4f89d71640 1 -- 172.21.15.92:0/163509414 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4f8409cda0 con 0x7f4f84099080 2024-09-21T09:43:04.206 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.202+0000 7f4f8a572640 1 --2- 172.21.15.92:0/163509414 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4f841346f0 0x7f4f8409c860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:04.206 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.202+0000 7f4f89d71640 1 --2- 172.21.15.92:0/163509414 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4f84099080 0x7f4f8409bde0 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f4f8000c530 tx=0x7f4f8000c9f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:04.206 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.202+0000 7f4f7a7fc640 1 -- 172.21.15.92:0/163509414 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f800052d0 con 0x7f4f84099080 2024-09-21T09:43:04.206 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.202+0000 7f4f7a7fc640 1 -- 172.21.15.92:0/163509414 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f4f80005470 con 0x7f4f84099080 2024-09-21T09:43:04.206 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.202+0000 7f4f8bfd5640 1 -- 172.21.15.92:0/163509414 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4f8409de80 con 0x7f4f84099080 2024-09-21T09:43:04.206 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.202+0000 7f4f7a7fc640 1 -- 172.21.15.92:0/163509414 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4f80012870 con 0x7f4f84099080 2024-09-21T09:43:04.206 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.202+0000 7f4f8bfd5640 1 -- 172.21.15.92:0/163509414 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4f8409d480 con 0x7f4f84099080 2024-09-21T09:43:04.206 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.203+0000 7f4f7a7fc640 1 -- 172.21.15.92:0/163509414 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f4f80012a10 con 0x7f4f84099080 2024-09-21T09:43:04.207 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.204+0000 7f4f89570640 1 -- 172.21.15.92:0/163509414 >> v1:172.21.15.92:6800/1280404317 conn(0x7f4f68047fd0 legacy=0x7f4f6804a490 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.92:6800/1280404317 2024-09-21T09:43:04.207 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.204+0000 7f4f7a7fc640 1 -- 172.21.15.92:0/163509414 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7f4f8005d4f0 con 0x7f4f84099080 2024-09-21T09:43:04.207 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.204+0000 7f4f8bfd5640 1 -- 172.21.15.92:0/163509414 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4f84099460 con 0x7f4f84099080 2024-09-21T09:43:04.209 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.207+0000 7f4f7a7fc640 1 -- 172.21.15.92:0/163509414 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+152684 (secure 0 0 0) 0x7f4f8005c680 con 0x7f4f84099080 2024-09-21T09:43:04.374 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.372+0000 7f4f8bfd5640 1 -- 172.21.15.92:0/163509414 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f4f840698c0 con 0x7f4f84099080 2024-09-21T09:43:04.375 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.373+0000 7f4f7a7fc640 1 -- 172.21.15.92:0/163509414 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v20) ==== 74+0+8812 (secure 0 0 0) 0x7f4f80041180 con 0x7f4f84099080 2024-09-21T09:43:04.375 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:04.375 INFO:teuthology.orchestra.run.smithi092.stdout:{"epoch":20,"fsid":"5f250e6c-4e36-49b1-ab62-d9454369fa03","created":"2024-09-21T09:41:34.680338+0000","modified":"2024-09-21T09:42:01.457080+0000","last_up_change":"2024-09-21T09:41:48.412163+0000","last_in_change":"2024-09-21T09:41:37.598778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"squid","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:41:50.040313+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:41:52.054798+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f3314f59-3374-4072-85c9-f8c4e458efc8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6805","nonce":195102377}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6806","nonce":195102377}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6808","nonce":195102377}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6807","nonce":195102377}]},"public_addr":"172.21.15.92:6805/195102377","cluster_addr":"172.21.15.92:6806/195102377","heartbeat_back_addr":"172.21.15.92:6808/195102377","heartbeat_front_addr":"172.21.15.92:6807/195102377","state":["exists","up"]},{"osd":1,"uuid":"cf0486af-ba68-4763-a364-c1f2181604a7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6813","nonce":1359151798}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6814","nonce":1359151798}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6816","nonce":1359151798}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6815","nonce":1359151798}]},"public_addr":"172.21.15.92:6813/1359151798","cluster_addr":"172.21.15.92:6814/1359151798","heartbeat_back_addr":"172.21.15.92:6816/1359151798","heartbeat_front_addr":"172.21.15.92:6815/1359151798","state":["exists","up"]},{"osd":2,"uuid":"5c587421-3409-4469-a2ee-b2ea20cd45e5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6809","nonce":296401696}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6810","nonce":296401696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6812","nonce":296401696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6811","nonce":296401696}]},"public_addr":"172.21.15.92:6809/296401696","cluster_addr":"172.21.15.92:6810/296401696","heartbeat_back_addr":"172.21.15.92:6812/296401696","heartbeat_front_addr":"172.21.15.92:6811/296401696","state":["exists","up"]},{"osd":3,"uuid":"b98663b7-de4d-4c12-ac91-8c33f159da35","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":15,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6801","nonce":1829519295}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6802","nonce":1829519295}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6804","nonce":1829519295}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6803","nonce":1829519295}]},"public_addr":"172.21.15.92:6801/1829519295","cluster_addr":"172.21.15.92:6802/1829519295","heartbeat_back_addr":"172.21.15.92:6804/1829519295","heartbeat_front_addr":"172.21.15.92:6803/1829519295","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.934317+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:45.144020+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.895848+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.636219+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:43:04.378 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.376+0000 7f4f8bfd5640 1 -- 172.21.15.92:0/163509414 >> v1:172.21.15.92:6800/1280404317 conn(0x7f4f68047fd0 legacy=0x7f4f6804a490 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:43:04.378 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.376+0000 7f4f8bfd5640 1 -- 172.21.15.92:0/163509414 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4f84099080 msgr2=0x7f4f8409bde0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.378 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.376+0000 7f4f8bfd5640 1 --2- 172.21.15.92:0/163509414 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4f84099080 0x7f4f8409bde0 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f4f8000c530 tx=0x7f4f8000c9f0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.378 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.376+0000 7f4f8bfd5640 1 -- 172.21.15.92:0/163509414 shutdown_connections 2024-09-21T09:43:04.378 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.376+0000 7f4f8bfd5640 1 --2- 172.21.15.92:0/163509414 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4f841346f0 0x7f4f8409c860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.378 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.376+0000 7f4f8bfd5640 1 --2- 172.21.15.92:0/163509414 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4f84131dc0 0x7f4f8409c320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.378 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.376+0000 7f4f8bfd5640 1 --2- 172.21.15.92:0/163509414 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4f84099080 0x7f4f8409bde0 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.379 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.376+0000 7f4f8bfd5640 1 -- 172.21.15.92:0/163509414 >> 172.21.15.92:0/163509414 conn(0x7f4f840a1800 msgr2=0x7f4f84135870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:04.379 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.376+0000 7f4f8bfd5640 1 -- 172.21.15.92:0/163509414 shutdown_connections 2024-09-21T09:43:04.379 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.376+0000 7f4f8bfd5640 1 -- 172.21.15.92:0/163509414 wait complete. 2024-09-21T09:43:04.393 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 flush_pg_stats 2024-09-21T09:43:04.394 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 flush_pg_stats 2024-09-21T09:43:04.394 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 flush_pg_stats 2024-09-21T09:43:04.394 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 flush_pg_stats 2024-09-21T09:43:04.510 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.508+0000 7f2c2fed5640 1 -- 172.21.15.92:0/1664259449 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2c2812f1c0 msgr2=0x7f2c2812f650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.510 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.508+0000 7f2c2fed5640 1 --2- 172.21.15.92:0/1664259449 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2c2812f1c0 0x7f2c2812f650 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f2c20009ca0 tx=0x7f2c2002a950 comp rx=0 tx=0).stop 2024-09-21T09:43:04.510 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.508+0000 7f2c2fed5640 1 -- 172.21.15.92:0/1664259449 shutdown_connections 2024-09-21T09:43:04.510 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.508+0000 7f2c2fed5640 1 --2- 172.21.15.92:0/1664259449 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2c2812f1c0 0x7f2c2812f650 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.510 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.508+0000 7f2c2fed5640 1 --2- 172.21.15.92:0/1664259449 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2c28098d40 0x7f2c28099120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.510 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.508+0000 7f2c2fed5640 1 -- 172.21.15.92:0/1664259449 >> 172.21.15.92:0/1664259449 conn(0x7f2c280a1790 msgr2=0x7f2c280a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:04.511 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.508+0000 7fd326758640 1 -- 172.21.15.92:0/494659463 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd3201309f0 msgr2=0x7fd320132de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.511 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.508+0000 7fd326758640 1 --2- 172.21.15.92:0/494659463 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd3201309f0 0x7fd320132de0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7fd318009c10 tx=0x7fd31802a780 comp rx=0 tx=0).stop 2024-09-21T09:43:04.511 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.508+0000 7fd326758640 1 -- 172.21.15.92:0/494659463 shutdown_connections 2024-09-21T09:43:04.511 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.508+0000 7fd326758640 1 --2- 172.21.15.92:0/494659463 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd3201309f0 0x7fd320132de0 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.511 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.508+0000 7fd326758640 1 --2- 172.21.15.92:0/494659463 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd3200702d0 0x7fd3201304b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.511 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.508+0000 7fd326758640 1 -- 172.21.15.92:0/494659463 >> 172.21.15.92:0/494659463 conn(0x7fd32009dde0 msgr2=0x7fd32009e1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:04.511 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.508+0000 7fd326758640 1 -- 172.21.15.92:0/494659463 shutdown_connections 2024-09-21T09:43:04.511 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.508+0000 7f2c2fed5640 1 -- 172.21.15.92:0/1664259449 shutdown_connections 2024-09-21T09:43:04.512 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.508+0000 7fd326758640 1 -- 172.21.15.92:0/494659463 wait complete. 2024-09-21T09:43:04.512 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.508+0000 7f2c2fed5640 1 -- 172.21.15.92:0/1664259449 wait complete. 2024-09-21T09:43:04.512 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.509+0000 7f2c2fed5640 1 Processor -- start 2024-09-21T09:43:04.512 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.509+0000 7fd326758640 1 Processor -- start 2024-09-21T09:43:04.513 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.509+0000 7f2c2fed5640 1 -- start start 2024-09-21T09:43:04.513 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.509+0000 7fd326758640 1 -- start start 2024-09-21T09:43:04.513 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.509+0000 7f2c2fed5640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2c28098d40 0x7f2c280713d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.514 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.510+0000 7f2c2fed5640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2c28099660 0x7f2c28071910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.514 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.510+0000 7f2c2fed5640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2c2812f1c0 0x7f2c28071e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.514 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.510+0000 7f2c2fed5640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f2c28137910 con 0x7f2c28099660 2024-09-21T09:43:04.514 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.510+0000 7f2c2fed5640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f2c28137790 con 0x7f2c2812f1c0 2024-09-21T09:43:04.514 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.510+0000 7f2c2fed5640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f2c28137a90 con 0x7f2c28098d40 2024-09-21T09:43:04.514 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.510+0000 7fd326758640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd32006f7e0 0x7fd320122a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.514 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.510+0000 7fd326758640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd3200702d0 0x7fd320122fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.514 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.510+0000 7fd326758640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd3201309f0 0x7fd320123500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.515 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.510+0000 7fd326758640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fd320133fc0 con 0x7fd3201309f0 2024-09-21T09:43:04.515 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.510+0000 7fd326758640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fd320133e40 con 0x7fd32006f7e0 2024-09-21T09:43:04.515 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.510+0000 7fd326758640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fd320134140 con 0x7fd3200702d0 2024-09-21T09:43:04.515 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.510+0000 7f2c2d470640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2c28099660 0x7f2c28071910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.515 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.510+0000 7f2c2dc71640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2c28098d40 0x7f2c280713d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.515 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.510+0000 7f2c2e472640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2c2812f1c0 0x7f2c28071e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.515 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.510+0000 7fd31f7fe640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd3200702d0 0x7fd320122fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.516 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.510+0000 7f2c2dc71640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2c28098d40 0x7f2c280713d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:60110/0 (socket says 172.21.15.92:60110) 2024-09-21T09:43:04.516 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.510+0000 7f2c2dc71640 1 -- 172.21.15.92:0/3054082853 learned_addr learned my addr 172.21.15.92:0/3054082853 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:04.517 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.510+0000 7fd324cf5640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd3201309f0 0x7fd320123500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.517 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.510+0000 7fd324cf5640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd3201309f0 0x7fd320123500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:48286/0 (socket says 172.21.15.92:48286) 2024-09-21T09:43:04.517 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.510+0000 7fd324cf5640 1 -- 172.21.15.92:0/3373242785 learned_addr learned my addr 172.21.15.92:0/3373242785 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:04.517 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.510+0000 7fd31ffff640 1 --2- 172.21.15.92:0/3373242785 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd32006f7e0 0x7fd320122a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.517 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.511+0000 7fd31ffff640 1 -- 172.21.15.92:0/3373242785 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd3200702d0 msgr2=0x7fd320122fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.517 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.511+0000 7fd31ffff640 1 --2- 172.21.15.92:0/3373242785 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd3200702d0 0x7fd320122fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.517 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.511+0000 7fd31ffff640 1 -- 172.21.15.92:0/3373242785 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd3201309f0 msgr2=0x7fd320123500 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.517 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.511+0000 7fd31ffff640 1 --2- 172.21.15.92:0/3373242785 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd3201309f0 0x7fd320123500 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.517 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.511+0000 7fd31ffff640 1 -- 172.21.15.92:0/3373242785 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd32012bce0 con 0x7fd32006f7e0 2024-09-21T09:43:04.518 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.511+0000 7f2c2d470640 1 -- 172.21.15.92:0/3054082853 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2c28098d40 msgr2=0x7f2c280713d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.518 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.511+0000 7f2c2d470640 1 --2- 172.21.15.92:0/3054082853 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2c28098d40 0x7f2c280713d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.518 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.511+0000 7f2c2d470640 1 -- 172.21.15.92:0/3054082853 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2c2812f1c0 msgr2=0x7f2c28071e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.518 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.511+0000 7f2c2d470640 1 --2- 172.21.15.92:0/3054082853 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2c2812f1c0 0x7f2c28071e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.518 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.511+0000 7f2c2d470640 1 -- 172.21.15.92:0/3054082853 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2c2812f5d0 con 0x7f2c28099660 2024-09-21T09:43:04.518 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.511+0000 7f2c2d470640 1 --2- 172.21.15.92:0/3054082853 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2c28099660 0x7f2c28071910 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f2c1800c430 tx=0x7f2c1800c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:04.519 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.511+0000 7fd31ffff640 1 --2- 172.21.15.92:0/3373242785 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd32006f7e0 0x7fd320122a80 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7fd31000c430 tx=0x7fd31000c8f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:04.519 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.512+0000 7f2c1e7fc640 1 -- 172.21.15.92:0/3054082853 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2c18017070 con 0x7f2c28099660 2024-09-21T09:43:04.519 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.512+0000 7f2c1e7fc640 1 -- 172.21.15.92:0/3054082853 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2c180053c0 con 0x7f2c28099660 2024-09-21T09:43:04.519 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.512+0000 7f2c2fed5640 1 -- 172.21.15.92:0/3054082853 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2c28072390 con 0x7f2c28099660 2024-09-21T09:43:04.520 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.512+0000 7fd31cff9640 1 -- 172.21.15.92:0/3373242785 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd310011790 con 0x7fd32006f7e0 2024-09-21T09:43:04.520 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.512+0000 7fd31cff9640 1 -- 172.21.15.92:0/3373242785 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fd310017070 con 0x7fd32006f7e0 2024-09-21T09:43:04.520 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.512+0000 7fd31cff9640 1 -- 172.21.15.92:0/3373242785 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd310011d00 con 0x7fd32006f7e0 2024-09-21T09:43:04.520 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.512+0000 7fd326758640 1 -- 172.21.15.92:0/3373242785 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd320124880 con 0x7fd32006f7e0 2024-09-21T09:43:04.520 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.512+0000 7fd326758640 1 -- 172.21.15.92:0/3373242785 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fd320097c30 con 0x7fd32006f7e0 2024-09-21T09:43:04.521 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.513+0000 7f2c1e7fc640 1 -- 172.21.15.92:0/3054082853 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2c18011890 con 0x7f2c28099660 2024-09-21T09:43:04.521 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.513+0000 7fd326758640 1 -- 172.21.15.92:0/3373242785 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_get_version(what=osdmap handle=1) -- 0x7fd32006fbc0 con 0x7fd32006f7e0 2024-09-21T09:43:04.521 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.513+0000 7f2c2fed5640 1 -- 172.21.15.92:0/3054082853 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2c28073dc0 con 0x7f2c28099660 2024-09-21T09:43:04.521 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.514+0000 7f2c1e7fc640 1 -- 172.21.15.92:0/3054082853 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f2c1801a070 con 0x7f2c28099660 2024-09-21T09:43:04.522 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.514+0000 7fd31cff9640 1 -- 172.21.15.92:0/3373242785 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fd31001c460 con 0x7fd32006f7e0 2024-09-21T09:43:04.522 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.514+0000 7fd31cff9640 1 -- 172.21.15.92:0/3373242785 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7fd31005d780 con 0x7fd32006f7e0 2024-09-21T09:43:04.522 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.514+0000 7fd31f7fe640 1 -- 172.21.15.92:0/3373242785 >> v1:172.21.15.92:6800/1280404317 conn(0x7fd30c0480a0 legacy=0x7fd30c04a560 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.92:6800/1280404317 2024-09-21T09:43:04.522 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.514+0000 7fd31cff9640 1 -- 172.21.15.92:0/3373242785 --> v1:172.21.15.92:6809/296401696 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7fd30c0523d0 con 0x7fd30c04e720 2024-09-21T09:43:04.522 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.514+0000 7fd31cff9640 1 -- 172.21.15.92:0/3373242785 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_get_version_reply(handle=1 version=20) ==== 24+0+0 (secure 0 0 0) 0x7fd310064050 con 0x7fd32006f7e0 2024-09-21T09:43:04.522 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.514+0000 7f2c0bfff640 1 -- 172.21.15.92:0/3054082853 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f2c28063590 con 0x7f2c28099660 2024-09-21T09:43:04.523 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.515+0000 7f2c1e7fc640 1 -- 172.21.15.92:0/3054082853 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7f2c1805edb0 con 0x7f2c28099660 2024-09-21T09:43:04.523 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.515+0000 7f2c1e7fc640 1 -- 172.21.15.92:0/3054082853 --> v1:172.21.15.92:6813/1359151798 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f2c0c052300 con 0x7f2c0c04e650 2024-09-21T09:43:04.523 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.515+0000 7f2c1e7fc640 1 -- 172.21.15.92:0/3054082853 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_get_version_reply(handle=1 version=20) ==== 24+0+0 (secure 0 0 0) 0x7f2c18067420 con 0x7f2c28099660 2024-09-21T09:43:04.523 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.515+0000 7f2c2dc71640 1 -- 172.21.15.92:0/3054082853 >> v1:172.21.15.92:6800/1280404317 conn(0x7f2c0c047fd0 legacy=0x7f2c0c04a490 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.92:6800/1280404317 2024-09-21T09:43:04.523 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.516+0000 7fd31cff9640 1 -- 172.21.15.92:0/3373242785 <== osd.2 v1:172.21.15.92:6809/296401696 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7fd30c0523d0 con 0x7fd30c04e720 2024-09-21T09:43:04.523 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.516+0000 7f2c1e7fc640 1 -- 172.21.15.92:0/3054082853 <== osd.1 v1:172.21.15.92:6813/1359151798 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f2c0c052300 con 0x7f2c0c04e650 2024-09-21T09:43:04.548 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.547+0000 7fd326758640 1 -- 172.21.15.92:0/3373242785 --> v1:172.21.15.92:6809/296401696 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7fd320123c90 con 0x7fd30c04e720 2024-09-21T09:43:04.549 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.547+0000 7fd31cff9640 1 -- 172.21.15.92:0/3373242785 <== osd.2 v1:172.21.15.92:6809/296401696 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 302421266) 0x7fd320123c90 con 0x7fd30c04e720 2024-09-21T09:43:04.549 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.548+0000 7f2c0bfff640 1 -- 172.21.15.92:0/3054082853 --> v1:172.21.15.92:6813/1359151798 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f2c280639c0 con 0x7f2c0c04e650 2024-09-21T09:43:04.550 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.548+0000 7fd326758640 1 -- 172.21.15.92:0/3373242785 >> v1:172.21.15.92:6809/296401696 conn(0x7fd30c04e720 legacy=0x7fd30c050b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.550 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.548+0000 7fd326758640 1 -- 172.21.15.92:0/3373242785 >> v1:172.21.15.92:6800/1280404317 conn(0x7fd30c0480a0 legacy=0x7fd30c04a560 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:43:04.550 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.548+0000 7f2c1e7fc640 1 -- 172.21.15.92:0/3054082853 <== osd.1 v1:172.21.15.92:6813/1359151798 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 4080920037) 0x7f2c280639c0 con 0x7f2c0c04e650 2024-09-21T09:43:04.550 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.548+0000 7fd326758640 1 -- 172.21.15.92:0/3373242785 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd32006f7e0 msgr2=0x7fd320122a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.550 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.548+0000 7fd326758640 1 --2- 172.21.15.92:0/3373242785 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd32006f7e0 0x7fd320122a80 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7fd31000c430 tx=0x7fd31000c8f0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.551 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.548+0000 7fd324cf5640 1 -- 172.21.15.92:0/3373242785 reap_dead start 2024-09-21T09:43:04.551 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.548+0000 7fd326758640 1 -- 172.21.15.92:0/3373242785 shutdown_connections 2024-09-21T09:43:04.551 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.548+0000 7fd326758640 1 -- 172.21.15.92:0/3373242785 >> 172.21.15.92:0/3373242785 conn(0x7fd32009dde0 msgr2=0x7fd3200760f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:04.551 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.548+0000 7fd326758640 1 -- 172.21.15.92:0/3373242785 shutdown_connections 2024-09-21T09:43:04.551 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.548+0000 7fd326758640 1 -- 172.21.15.92:0/3373242785 wait complete. 2024-09-21T09:43:04.552 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.548+0000 7f2c0bfff640 1 -- 172.21.15.92:0/3054082853 >> v1:172.21.15.92:6813/1359151798 conn(0x7f2c0c04e650 legacy=0x7f2c0c050a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.552 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.549+0000 7f2c0bfff640 1 -- 172.21.15.92:0/3054082853 >> v1:172.21.15.92:6800/1280404317 conn(0x7f2c0c047fd0 legacy=0x7f2c0c04a490 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:43:04.552 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.549+0000 7f2c0bfff640 1 -- 172.21.15.92:0/3054082853 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2c28099660 msgr2=0x7f2c28071910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.552 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.549+0000 7f2c0bfff640 1 --2- 172.21.15.92:0/3054082853 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2c28099660 0x7f2c28071910 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f2c1800c430 tx=0x7f2c1800c8f0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.552 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.549+0000 7f2c2e472640 1 -- 172.21.15.92:0/3054082853 reap_dead start 2024-09-21T09:43:04.552 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.549+0000 7f2c0bfff640 1 -- 172.21.15.92:0/3054082853 shutdown_connections 2024-09-21T09:43:04.552 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.549+0000 7f2c0bfff640 1 -- 172.21.15.92:0/3054082853 >> 172.21.15.92:0/3054082853 conn(0x7f2c280a1790 msgr2=0x7f2c2809cf10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:04.552 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.549+0000 7f2c0bfff640 1 -- 172.21.15.92:0/3054082853 shutdown_connections 2024-09-21T09:43:04.552 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.549+0000 7f2c0bfff640 1 -- 172.21.15.92:0/3054082853 wait complete. 2024-09-21T09:43:04.553 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.551+0000 7f12dd298640 1 -- 172.21.15.92:0/77640779 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f12d806f7e0 msgr2=0x7f12d806fbc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.553 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.551+0000 7f12dd298640 1 --2- 172.21.15.92:0/77640779 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f12d806f7e0 0x7f12d806fbc0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f12c8009860 tx=0x7f12c802c5a0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.553 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.551+0000 7f12dd298640 1 -- 172.21.15.92:0/77640779 shutdown_connections 2024-09-21T09:43:04.554 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.551+0000 7f12dd298640 1 --2- 172.21.15.92:0/77640779 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f12d812b6b0 0x7f12d812bb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.554 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.551+0000 7f12dd298640 1 --2- 172.21.15.92:0/77640779 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f12d8070100 0x7f12d812b170 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.554 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.551+0000 7f12dd298640 1 --2- 172.21.15.92:0/77640779 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f12d806f7e0 0x7f12d806fbc0 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.554 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.551+0000 7f12dd298640 1 -- 172.21.15.92:0/77640779 >> 172.21.15.92:0/77640779 conn(0x7f12d809dde0 msgr2=0x7f12d809e1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:04.554 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.551+0000 7f12dd298640 1 -- 172.21.15.92:0/77640779 shutdown_connections 2024-09-21T09:43:04.554 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.552+0000 7f12dd298640 1 -- 172.21.15.92:0/77640779 wait complete. 2024-09-21T09:43:04.554 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.553+0000 7f12dd298640 1 Processor -- start 2024-09-21T09:43:04.554 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.553+0000 7f12dd298640 1 -- start start 2024-09-21T09:43:04.556 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.554+0000 7f12dd298640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f12d806f7e0 0x7f12d812d3e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.556 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.554+0000 7f12dd298640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f12d8070100 0x7f12d812d920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.556 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.554+0000 7f12dd298640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f12d812b6b0 0x7f12d8130960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.556 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.554+0000 7f12d67fc640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f12d8070100 0x7f12d812d920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.556 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.554+0000 7f12d6ffd640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f12d806f7e0 0x7f12d812d3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.556 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.554+0000 7f12d6ffd640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f12d806f7e0 0x7f12d812d3e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:34764/0 (socket says 172.21.15.92:34764) 2024-09-21T09:43:04.556 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.554+0000 7f12d6ffd640 1 -- 172.21.15.92:0/640243137 learned_addr learned my addr 172.21.15.92:0/640243137 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:04.556 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.554+0000 7f12dd298640 1 -- 172.21.15.92:0/640243137 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f12d81324c0 con 0x7f12d8070100 2024-09-21T09:43:04.557 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.554+0000 7f12dd298640 1 -- 172.21.15.92:0/640243137 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f12d8076770 con 0x7f12d806f7e0 2024-09-21T09:43:04.557 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.554+0000 7f12dd298640 1 -- 172.21.15.92:0/640243137 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f12d8132640 con 0x7f12d812b6b0 2024-09-21T09:43:04.557 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.554+0000 7f12d77fe640 1 --2- 172.21.15.92:0/640243137 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f12d812b6b0 0x7f12d8130960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.558 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.555+0000 7f12d77fe640 1 -- 172.21.15.92:0/640243137 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f12d806f7e0 msgr2=0x7f12d812d3e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.558 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.555+0000 7f12d77fe640 1 --2- 172.21.15.92:0/640243137 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f12d806f7e0 0x7f12d812d3e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.558 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.555+0000 7f12d77fe640 1 -- 172.21.15.92:0/640243137 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f12d8070100 msgr2=0x7f12d812d920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.558 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.555+0000 7f12d77fe640 1 --2- 172.21.15.92:0/640243137 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f12d8070100 0x7f12d812d920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.558 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.555+0000 7f12d77fe640 1 -- 172.21.15.92:0/640243137 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f12d81306a0 con 0x7f12d812b6b0 2024-09-21T09:43:04.558 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.555+0000 7f12d6ffd640 1 --2- 172.21.15.92:0/640243137 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f12d806f7e0 0x7f12d812d3e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T09:43:04.558 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.555+0000 7f12d67fc640 1 --2- 172.21.15.92:0/640243137 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f12d8070100 0x7f12d812d920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:04.558 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.555+0000 7f12d77fe640 1 --2- 172.21.15.92:0/640243137 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f12d812b6b0 0x7f12d8130960 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f12d0005fd0 tx=0x7f12d000ef60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:04.559 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.555+0000 7f12bf7fe640 1 -- 172.21.15.92:0/640243137 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f12d0004b20 con 0x7f12d812b6b0 2024-09-21T09:43:04.559 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.555+0000 7f12dd298640 1 -- 172.21.15.92:0/640243137 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f12d812f3b0 con 0x7f12d812b6b0 2024-09-21T09:43:04.559 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.555+0000 7f12bf7fe640 1 -- 172.21.15.92:0/640243137 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f12d0004cc0 con 0x7f12d812b6b0 2024-09-21T09:43:04.559 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.555+0000 7f12dd298640 1 -- 172.21.15.92:0/640243137 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f12d812e040 con 0x7f12d812b6b0 2024-09-21T09:43:04.559 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.556+0000 7f12bf7fe640 1 -- 172.21.15.92:0/640243137 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f12d0005280 con 0x7f12d812b6b0 2024-09-21T09:43:04.559 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.557+0000 7f12bf7fe640 1 -- 172.21.15.92:0/640243137 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f12d0005420 con 0x7f12d812b6b0 2024-09-21T09:43:04.560 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.556+0000 7fcd155d5640 1 -- 172.21.15.92:0/4001892582 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcd100974a0 msgr2=0x7fcd1012be30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.560 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.556+0000 7fcd155d5640 1 --2- 172.21.15.92:0/4001892582 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcd100974a0 0x7fcd1012be30 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7fcd08009f00 tx=0x7fcd0802ac20 comp rx=0 tx=0).stop 2024-09-21T09:43:04.560 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.557+0000 7fcd155d5640 1 -- 172.21.15.92:0/4001892582 shutdown_connections 2024-09-21T09:43:04.560 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.557+0000 7fcd155d5640 1 --2- 172.21.15.92:0/4001892582 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcd100974a0 0x7fcd1012be30 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.560 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.557+0000 7fcd155d5640 1 --2- 172.21.15.92:0/4001892582 >> v2:172.21.15.92:3300/0 conn(0x7fcd10095e10 0x7fcd100961f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.560 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.557+0000 7fcd155d5640 1 -- 172.21.15.92:0/4001892582 >> 172.21.15.92:0/4001892582 conn(0x7fcd1009c470 msgr2=0x7fcd1009c880 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:04.560 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.557+0000 7fcd155d5640 1 -- 172.21.15.92:0/4001892582 shutdown_connections 2024-09-21T09:43:04.560 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.557+0000 7fcd155d5640 1 -- 172.21.15.92:0/4001892582 wait complete. 2024-09-21T09:43:04.561 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.557+0000 7f12dd298640 1 -- 172.21.15.92:0/640243137 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f12b8001030 con 0x7f12d812b6b0 2024-09-21T09:43:04.561 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.558+0000 7f12d6ffd640 1 -- 172.21.15.92:0/640243137 >> v1:172.21.15.92:6800/1280404317 conn(0x7f12c4047fd0 legacy=0x7f12c404a490 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.92:6800/1280404317 2024-09-21T09:43:04.561 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.558+0000 7f12bf7fe640 1 -- 172.21.15.92:0/640243137 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7f12d00673d0 con 0x7f12d812b6b0 2024-09-21T09:43:04.561 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.558+0000 7f12bf7fe640 1 -- 172.21.15.92:0/640243137 --> v1:172.21.15.92:6805/195102377 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f12d0013e50 con 0x7f12c404e6c0 2024-09-21T09:43:04.561 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.558+0000 7f12bf7fe640 1 -- 172.21.15.92:0/640243137 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_get_version_reply(handle=1 version=20) ==== 24+0+0 (secure 0 0 0) 0x7f12d006d050 con 0x7f12d812b6b0 2024-09-21T09:43:04.562 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.558+0000 7fcd155d5640 1 Processor -- start 2024-09-21T09:43:04.562 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.558+0000 7fcd155d5640 1 -- start start 2024-09-21T09:43:04.562 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.559+0000 7fcd155d5640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcd10095e10 0x7fcd10092170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.562 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.559+0000 7fcd155d5640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcd10096730 0x7fcd100926b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.562 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.559+0000 7fcd0effd640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcd10095e10 0x7fcd10092170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.562 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.559+0000 7fcd0effd640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcd10095e10 0x7fcd10092170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:60144/0 (socket says 172.21.15.92:60144) 2024-09-21T09:43:04.563 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.559+0000 7fcd0e7fc640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcd10096730 0x7fcd100926b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.563 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.559+0000 7fcd0e7fc640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcd10096730 0x7fcd100926b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:48342/0 (socket says 172.21.15.92:48342) 2024-09-21T09:43:04.563 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.559+0000 7fcd0effd640 1 -- 172.21.15.92:0/3746842639 learned_addr learned my addr 172.21.15.92:0/3746842639 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:04.563 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.559+0000 7fcd155d5640 1 --2- 172.21.15.92:0/3746842639 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcd100974a0 0x7fcd10092bf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.563 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.559+0000 7f12bf7fe640 1 -- 172.21.15.92:0/640243137 <== osd.0 v1:172.21.15.92:6805/195102377 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f12d0013e50 con 0x7f12c404e6c0 2024-09-21T09:43:04.565 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.560+0000 7fcd0f7fe640 1 --2- 172.21.15.92:0/3746842639 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcd100974a0 0x7fcd10092bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.565 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.560+0000 7fcd155d5640 1 -- 172.21.15.92:0/3746842639 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fcd10134580 con 0x7fcd10096730 2024-09-21T09:43:04.565 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.560+0000 7fcd155d5640 1 -- 172.21.15.92:0/3746842639 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fcd10134400 con 0x7fcd100974a0 2024-09-21T09:43:04.565 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.560+0000 7fcd155d5640 1 -- 172.21.15.92:0/3746842639 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fcd10134700 con 0x7fcd10095e10 2024-09-21T09:43:04.565 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.560+0000 7fcd0effd640 1 -- 172.21.15.92:0/3746842639 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcd100974a0 msgr2=0x7fcd10092bf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.565 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.560+0000 7fcd0effd640 1 --2- 172.21.15.92:0/3746842639 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcd100974a0 0x7fcd10092bf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.566 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.560+0000 7fcd0effd640 1 -- 172.21.15.92:0/3746842639 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcd10096730 msgr2=0x7fcd100926b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.566 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.560+0000 7fcd0effd640 1 --2- 172.21.15.92:0/3746842639 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcd10096730 0x7fcd100926b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.566 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.560+0000 7fcd0effd640 1 -- 172.21.15.92:0/3746842639 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcd100946d0 con 0x7fcd10095e10 2024-09-21T09:43:04.566 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.560+0000 7fcd0f7fe640 1 --2- 172.21.15.92:0/3746842639 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcd100974a0 0x7fcd10092bf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:43:04.566 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.560+0000 7fcd0e7fc640 1 --2- 172.21.15.92:0/3746842639 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcd10096730 0x7fcd100926b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:04.566 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.560+0000 7fcd0effd640 1 --2- 172.21.15.92:0/3746842639 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcd10095e10 0x7fcd10092170 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7fcd0000e9f0 tx=0x7fcd0000eeb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:04.566 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.561+0000 7fccf77fe640 1 -- 172.21.15.92:0/3746842639 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcd0000cd80 con 0x7fcd10095e10 2024-09-21T09:43:04.566 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.561+0000 7fcd155d5640 1 -- 172.21.15.92:0/3746842639 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fcd10094e40 con 0x7fcd10095e10 2024-09-21T09:43:04.566 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.561+0000 7fcd155d5640 1 -- 172.21.15.92:0/3746842639 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fcd10093310 con 0x7fcd10095e10 2024-09-21T09:43:04.566 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.561+0000 7fccf77fe640 1 -- 172.21.15.92:0/3746842639 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fcd00010430 con 0x7fcd10095e10 2024-09-21T09:43:04.566 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.561+0000 7fccf77fe640 1 -- 172.21.15.92:0/3746842639 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcd00026060 con 0x7fcd10095e10 2024-09-21T09:43:04.566 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.562+0000 7fcd155d5640 1 -- 172.21.15.92:0/3746842639 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_get_version(what=osdmap handle=1) -- 0x7fcd10063590 con 0x7fcd10095e10 2024-09-21T09:43:04.566 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.563+0000 7fccf77fe640 1 -- 172.21.15.92:0/3746842639 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fcd000107e0 con 0x7fcd10095e10 2024-09-21T09:43:04.567 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.563+0000 7fcd0e7fc640 1 -- 172.21.15.92:0/3746842639 >> v1:172.21.15.92:6800/1280404317 conn(0x7fccfc047fd0 legacy=0x7fccfc04a490 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.92:6800/1280404317 2024-09-21T09:43:04.567 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.563+0000 7fccf77fe640 1 -- 172.21.15.92:0/3746842639 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7fcd0005e4c0 con 0x7fcd10095e10 2024-09-21T09:43:04.567 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.563+0000 7fccf77fe640 1 -- 172.21.15.92:0/3746842639 --> v1:172.21.15.92:6801/1829519295 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7fcd00023e50 con 0x7fccfc04e650 2024-09-21T09:43:04.567 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.563+0000 7fccf77fe640 1 -- 172.21.15.92:0/3746842639 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_get_version_reply(handle=1 version=20) ==== 24+0+0 (secure 0 0 0) 0x7fcd00065050 con 0x7fcd10095e10 2024-09-21T09:43:04.567 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.565+0000 7fccf77fe640 1 -- 172.21.15.92:0/3746842639 <== osd.3 v1:172.21.15.92:6801/1829519295 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7fcd00023e50 con 0x7fccfc04e650 2024-09-21T09:43:04.568 INFO:teuthology.orchestra.run.smithi092.stdout:47244640276 2024-09-21T09:43:04.568 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T09:43:04.571 INFO:teuthology.orchestra.run.smithi092.stdout:47244640274 2024-09-21T09:43:04.571 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T09:43:04.579 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.577+0000 7f12dd298640 1 -- 172.21.15.92:0/640243137 --> v1:172.21.15.92:6805/195102377 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f12b8002dd0 con 0x7f12c404e6c0 2024-09-21T09:43:04.581 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.577+0000 7f12bf7fe640 1 -- 172.21.15.92:0/640243137 <== osd.0 v1:172.21.15.92:6805/195102377 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 302421266) 0x7f12b8002dd0 con 0x7f12c404e6c0 2024-09-21T09:43:04.582 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.578+0000 7f12dd298640 1 -- 172.21.15.92:0/640243137 >> v1:172.21.15.92:6805/195102377 conn(0x7f12c404e6c0 legacy=0x7f12c4050b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.583 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.578+0000 7f12dd298640 1 -- 172.21.15.92:0/640243137 >> v1:172.21.15.92:6800/1280404317 conn(0x7f12c4047fd0 legacy=0x7f12c404a490 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:43:04.583 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.578+0000 7f12dd298640 1 -- 172.21.15.92:0/640243137 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f12d812b6b0 msgr2=0x7f12d8130960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.583 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.578+0000 7f12dd298640 1 --2- 172.21.15.92:0/640243137 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f12d812b6b0 0x7f12d8130960 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f12d0005fd0 tx=0x7f12d000ef60 comp rx=0 tx=0).stop 2024-09-21T09:43:04.583 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.578+0000 7f12d77fe640 1 -- 172.21.15.92:0/640243137 reap_dead start 2024-09-21T09:43:04.583 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.578+0000 7f12dd298640 1 -- 172.21.15.92:0/640243137 shutdown_connections 2024-09-21T09:43:04.584 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.579+0000 7f12dd298640 1 -- 172.21.15.92:0/640243137 >> 172.21.15.92:0/640243137 conn(0x7f12d809dde0 msgr2=0x7f12d8122a80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:04.584 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.579+0000 7f12dd298640 1 -- 172.21.15.92:0/640243137 shutdown_connections 2024-09-21T09:43:04.584 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.579+0000 7f12dd298640 1 -- 172.21.15.92:0/640243137 wait complete. 2024-09-21T09:43:04.595 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.594+0000 7fcd155d5640 1 -- 172.21.15.92:0/3746842639 --> v1:172.21.15.92:6801/1829519295 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7fcd10063990 con 0x7fccfc04e650 2024-09-21T09:43:04.596 INFO:teuthology.orchestra.run.smithi092.stdout:47244640276 2024-09-21T09:43:04.596 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T09:43:04.597 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.594+0000 7fccf77fe640 1 -- 172.21.15.92:0/3746842639 <== osd.3 v1:172.21.15.92:6801/1829519295 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 3077695196) 0x7fcd10063990 con 0x7fccfc04e650 2024-09-21T09:43:04.598 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.595+0000 7fcd155d5640 1 -- 172.21.15.92:0/3746842639 >> v1:172.21.15.92:6801/1829519295 conn(0x7fccfc04e650 legacy=0x7fccfc050a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.598 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.595+0000 7fcd155d5640 1 -- 172.21.15.92:0/3746842639 >> v1:172.21.15.92:6800/1280404317 conn(0x7fccfc047fd0 legacy=0x7fccfc04a490 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:43:04.598 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.595+0000 7fcd155d5640 1 -- 172.21.15.92:0/3746842639 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcd10095e10 msgr2=0x7fcd10092170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.598 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.595+0000 7fcd155d5640 1 --2- 172.21.15.92:0/3746842639 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcd10095e10 0x7fcd10092170 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7fcd0000e9f0 tx=0x7fcd0000eeb0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.598 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.595+0000 7fcd0f7fe640 1 -- 172.21.15.92:0/3746842639 reap_dead start 2024-09-21T09:43:04.598 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.595+0000 7fcd155d5640 1 -- 172.21.15.92:0/3746842639 shutdown_connections 2024-09-21T09:43:04.598 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.595+0000 7fcd155d5640 1 -- 172.21.15.92:0/3746842639 >> 172.21.15.92:0/3746842639 conn(0x7fcd1009c470 msgr2=0x7fcd10133010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:04.598 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.595+0000 7fcd155d5640 1 -- 172.21.15.92:0/3746842639 shutdown_connections 2024-09-21T09:43:04.598 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.595+0000 7fcd155d5640 1 -- 172.21.15.92:0/3746842639 wait complete. 2024-09-21T09:43:04.613 INFO:teuthology.orchestra.run.smithi092.stdout:38654705683 2024-09-21T09:43:04.613 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T09:43:04.687 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.685+0000 7f0672e52640 1 -- 172.21.15.92:0/431803144 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f066c0916e0 msgr2=0x7f066c134ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.687 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.685+0000 7f0672e52640 1 --2- 172.21.15.92:0/431803144 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f066c0916e0 0x7f066c134ed0 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f0664005fd0 tx=0x7f066402a910 comp rx=0 tx=0).stop 2024-09-21T09:43:04.687 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.685+0000 7f0672e52640 1 -- 172.21.15.92:0/431803144 shutdown_connections 2024-09-21T09:43:04.687 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.685+0000 7f0672e52640 1 --2- 172.21.15.92:0/431803144 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f066c0916e0 0x7f066c134ed0 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.687 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.685+0000 7f0672e52640 1 --2- 172.21.15.92:0/431803144 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f066c12c980 0x7f066c12cd60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.688 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.685+0000 7f0672e52640 1 -- 172.21.15.92:0/431803144 >> 172.21.15.92:0/431803144 conn(0x7f066c09c470 msgr2=0x7f066c09c880 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:04.688 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.685+0000 7f0672e52640 1 -- 172.21.15.92:0/431803144 shutdown_connections 2024-09-21T09:43:04.688 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.685+0000 7f0672e52640 1 -- 172.21.15.92:0/431803144 wait complete. 2024-09-21T09:43:04.690 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.686+0000 7f0672e52640 1 Processor -- start 2024-09-21T09:43:04.690 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.686+0000 7f0672e52640 1 -- start start 2024-09-21T09:43:04.690 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.687+0000 7f0672e52640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f066c0916e0 0x7f066c06f710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.690 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.687+0000 7f0672e52640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f066c12c980 0x7f066c06fc50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.690 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.687+0000 7f0672e52640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f066c12d2a0 0x7f066c070190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.690 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.687+0000 7f0672e52640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f066c1357a0 con 0x7f066c12c980 2024-09-21T09:43:04.690 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.687+0000 7f0672e52640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f066c135620 con 0x7f066c0916e0 2024-09-21T09:43:04.690 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.687+0000 7f0672e52640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f066c135920 con 0x7f066c12d2a0 2024-09-21T09:43:04.690 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.687+0000 7f066bfff640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f066c12c980 0x7f066c06fc50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.690 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.687+0000 7f0670bee640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f066c0916e0 0x7f066c06f710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.690 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.687+0000 7f0670bee640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f066c0916e0 0x7f066c06f710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:34790/0 (socket says 172.21.15.92:34790) 2024-09-21T09:43:04.690 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.687+0000 7f0670bee640 1 -- 172.21.15.92:0/1140450962 learned_addr learned my addr 172.21.15.92:0/1140450962 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:04.691 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.687+0000 7f06713ef640 1 --2- 172.21.15.92:0/1140450962 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f066c12d2a0 0x7f066c070190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.691 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.688+0000 7f0670bee640 1 -- 172.21.15.92:0/1140450962 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f066c12d2a0 msgr2=0x7f066c070190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.691 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.688+0000 7f0670bee640 1 --2- 172.21.15.92:0/1140450962 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f066c12d2a0 0x7f066c070190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.691 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.688+0000 7f0670bee640 1 -- 172.21.15.92:0/1140450962 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f066c12c980 msgr2=0x7f066c06fc50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.691 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.688+0000 7f0670bee640 1 --2- 172.21.15.92:0/1140450962 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f066c12c980 0x7f066c06fc50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.691 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.688+0000 7f0670bee640 1 -- 172.21.15.92:0/1140450962 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f066c126450 con 0x7f066c0916e0 2024-09-21T09:43:04.691 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.688+0000 7f0670bee640 1 --2- 172.21.15.92:0/1140450962 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f066c0916e0 0x7f066c06f710 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f066000d950 tx=0x7f066000de10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:04.692 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.688+0000 7f06697fa640 1 -- 172.21.15.92:0/1140450962 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0660017070 con 0x7f066c0916e0 2024-09-21T09:43:04.692 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.688+0000 7f06697fa640 1 -- 172.21.15.92:0/1140450962 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f0660009d20 con 0x7f066c0916e0 2024-09-21T09:43:04.692 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.688+0000 7f0672e52640 1 -- 172.21.15.92:0/1140450962 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f066c127250 con 0x7f066c0916e0 2024-09-21T09:43:04.692 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.689+0000 7f06697fa640 1 -- 172.21.15.92:0/1140450962 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f06600106c0 con 0x7f066c0916e0 2024-09-21T09:43:04.692 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.689+0000 7f0672e52640 1 -- 172.21.15.92:0/1140450962 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f066c1c8b00 con 0x7f066c0916e0 2024-09-21T09:43:04.692 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.690+0000 7f0656ffd640 1 -- 172.21.15.92:0/1140450962 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f066c12cd60 con 0x7f066c0916e0 2024-09-21T09:43:04.693 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.691+0000 7f06697fa640 1 -- 172.21.15.92:0/1140450962 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f0660010860 con 0x7f066c0916e0 2024-09-21T09:43:04.693 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.692+0000 7f066bfff640 1 -- 172.21.15.92:0/1140450962 >> v1:172.21.15.92:6800/1280404317 conn(0x7f0650047fd0 legacy=0x7f065004a490 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.92:6800/1280404317 2024-09-21T09:43:04.693 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.692+0000 7f06697fa640 1 -- 172.21.15.92:0/1140450962 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7f066003d4b0 con 0x7f066c0916e0 2024-09-21T09:43:04.697 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.695+0000 7f06697fa640 1 -- 172.21.15.92:0/1140450962 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+152684 (secure 0 0 0) 0x7f066003ef50 con 0x7f066c0916e0 2024-09-21T09:43:04.722 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.719+0000 7fab5b628640 1 -- 172.21.15.92:0/806605135 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fab54129e30 msgr2=0x7fab5412c820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.722 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.720+0000 7fab5b628640 1 --2- 172.21.15.92:0/806605135 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fab54129e30 0x7fab5412c820 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7fab44009830 tx=0x7fab4402a3f0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.722 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.720+0000 7fab5b628640 1 -- 172.21.15.92:0/806605135 shutdown_connections 2024-09-21T09:43:04.722 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.720+0000 7fab5b628640 1 --2- 172.21.15.92:0/806605135 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fab54129e30 0x7fab5412c820 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.722 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.720+0000 7fab5b628640 1 --2- 172.21.15.92:0/806605135 >> v2:172.21.15.92:3300/0 conn(0x7fab54129510 0x7fab541298f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.722 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.720+0000 7fab5b628640 1 -- 172.21.15.92:0/806605135 >> 172.21.15.92:0/806605135 conn(0x7fab5409d370 msgr2=0x7fab5409d780 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:04.722 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.720+0000 7fab5b628640 1 -- 172.21.15.92:0/806605135 shutdown_connections 2024-09-21T09:43:04.722 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.720+0000 7fab5b628640 1 -- 172.21.15.92:0/806605135 wait complete. 2024-09-21T09:43:04.723 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.722+0000 7fab5b628640 1 Processor -- start 2024-09-21T09:43:04.723 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.722+0000 7fab5b628640 1 -- start start 2024-09-21T09:43:04.724 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.723+0000 7fab5b628640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fab54129510 0x7fab54130610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.725 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.723+0000 7fab5b628640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fab54129e30 0x7fab54130b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.725 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.723+0000 7fab593c4640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fab54129510 0x7fab54130610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.725 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.723+0000 7fab593c4640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fab54129510 0x7fab54130610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:48376/0 (socket says 172.21.15.92:48376) 2024-09-21T09:43:04.725 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.723+0000 7fab58bc3640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fab54129e30 0x7fab54130b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.726 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.723+0000 7fab58bc3640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fab54129e30 0x7fab54130b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:34800/0 (socket says 172.21.15.92:34800) 2024-09-21T09:43:04.726 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.723+0000 7fab593c4640 1 -- 172.21.15.92:0/772643251 learned_addr learned my addr 172.21.15.92:0/772643251 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:04.726 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.723+0000 7fab5b628640 1 --2- 172.21.15.92:0/772643251 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fab5412cd60 0x7fab54131090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.726 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.723+0000 7fab59bc5640 1 --2- 172.21.15.92:0/772643251 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fab5412cd60 0x7fab54131090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.726 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.724+0000 7fab5b628640 1 -- 172.21.15.92:0/772643251 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fab541357e0 con 0x7fab54129510 2024-09-21T09:43:04.726 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.724+0000 7fab5b628640 1 -- 172.21.15.92:0/772643251 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fab54135660 con 0x7fab54129e30 2024-09-21T09:43:04.726 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.724+0000 7fab5b628640 1 -- 172.21.15.92:0/772643251 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fab54135960 con 0x7fab5412cd60 2024-09-21T09:43:04.727 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.724+0000 7fab593c4640 1 -- 172.21.15.92:0/772643251 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fab5412cd60 msgr2=0x7fab54131090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.727 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.724+0000 7fab593c4640 1 --2- 172.21.15.92:0/772643251 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fab5412cd60 0x7fab54131090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.727 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.725+0000 7fab593c4640 1 -- 172.21.15.92:0/772643251 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fab54129e30 msgr2=0x7fab54130b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.727 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.725+0000 7fab593c4640 1 --2- 172.21.15.92:0/772643251 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fab54129e30 0x7fab54130b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.728 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.725+0000 7fab593c4640 1 -- 172.21.15.92:0/772643251 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fab54133840 con 0x7fab54129510 2024-09-21T09:43:04.728 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.725+0000 7fab58bc3640 1 --2- 172.21.15.92:0/772643251 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fab54129e30 0x7fab54130b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:04.728 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.725+0000 7fab59bc5640 1 --2- 172.21.15.92:0/772643251 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fab5412cd60 0x7fab54131090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:43:04.728 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.725+0000 7fab593c4640 1 --2- 172.21.15.92:0/772643251 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fab54129510 0x7fab54130610 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7fab5000d950 tx=0x7fab5000de10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:04.728 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.725+0000 7fab49ffb640 1 -- 172.21.15.92:0/772643251 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fab50017070 con 0x7fab54129510 2024-09-21T09:43:04.728 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.725+0000 7fab49ffb640 1 -- 172.21.15.92:0/772643251 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fab50009d20 con 0x7fab54129510 2024-09-21T09:43:04.728 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.725+0000 7fab49ffb640 1 -- 172.21.15.92:0/772643251 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fab500106c0 con 0x7fab54129510 2024-09-21T09:43:04.728 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.726+0000 7fab5b628640 1 -- 172.21.15.92:0/772643251 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fab54131e60 con 0x7fab54129510 2024-09-21T09:43:04.729 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.727+0000 7fab5b628640 1 -- 172.21.15.92:0/772643251 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fab54132490 con 0x7fab54129510 2024-09-21T09:43:04.729 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.727+0000 7fab49ffb640 1 -- 172.21.15.92:0/772643251 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fab50010860 con 0x7fab54129510 2024-09-21T09:43:04.730 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.728+0000 7fab58bc3640 1 -- 172.21.15.92:0/772643251 >> v1:172.21.15.92:6800/1280404317 conn(0x7fab3804c3e0 legacy=0x7fab3804e8a0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.92:6800/1280404317 2024-09-21T09:43:04.730 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.728+0000 7fab49ffb640 1 -- 172.21.15.92:0/772643251 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7fab50062cd0 con 0x7fab54129510 2024-09-21T09:43:04.730 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.728+0000 7fab5b628640 1 -- 172.21.15.92:0/772643251 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fab541298f0 con 0x7fab54129510 2024-09-21T09:43:04.733 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.732+0000 7fab49ffb640 1 -- 172.21.15.92:0/772643251 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+152684 (secure 0 0 0) 0x7fab54133840 con 0x7fab54129510 2024-09-21T09:43:04.739 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.737+0000 7ff7ae549640 1 -- 172.21.15.92:0/1398769179 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff7a8091510 msgr2=0x7ff7a80919c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.737+0000 7ff7ae549640 1 --2- 172.21.15.92:0/1398769179 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff7a8091510 0x7ff7a80919c0 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7ff7a0009f00 tx=0x7ff7a002ab90 comp rx=0 tx=0).stop 2024-09-21T09:43:04.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.737+0000 7ff7ae549640 1 -- 172.21.15.92:0/1398769179 shutdown_connections 2024-09-21T09:43:04.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.737+0000 7ff7ae549640 1 --2- 172.21.15.92:0/1398769179 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff7a8091510 0x7ff7a80919c0 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.737+0000 7ff7ae549640 1 --2- 172.21.15.92:0/1398769179 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff7a806ebf0 0x7ff7a8090fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.737+0000 7ff7ae549640 1 --2- 172.21.15.92:0/1398769179 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff7a806e2d0 0x7ff7a806e6b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.737+0000 7ff7ae549640 1 -- 172.21.15.92:0/1398769179 >> 172.21.15.92:0/1398769179 conn(0x7ff7a809c230 msgr2=0x7ff7a809c640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:04.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.737+0000 7ff7ae549640 1 -- 172.21.15.92:0/1398769179 shutdown_connections 2024-09-21T09:43:04.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.738+0000 7ff7ae549640 1 -- 172.21.15.92:0/1398769179 wait complete. 2024-09-21T09:43:04.741 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.739+0000 7ff7ae549640 1 Processor -- start 2024-09-21T09:43:04.741 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.739+0000 7ff7ae549640 1 -- start start 2024-09-21T09:43:04.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.740+0000 7ff7ae549640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff7a806e2d0 0x7ff7a8132fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.740+0000 7ff7ae549640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff7a806ebf0 0x7ff7a8133500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.740+0000 7ff7ae549640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff7a8091510 0x7ff7a8133a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.740+0000 7ff7ae549640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7ff7a81346f0 con 0x7ff7a8091510 2024-09-21T09:43:04.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.740+0000 7ff7ae549640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7ff7a8134570 con 0x7ff7a806e2d0 2024-09-21T09:43:04.744 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.740+0000 7ff7ae549640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7ff7a8134870 con 0x7ff7a806ebf0 2024-09-21T09:43:04.744 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.740+0000 7ff7a7fff640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff7a806e2d0 0x7ff7a8132fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.744 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.740+0000 7ff7a77fe640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff7a806ebf0 0x7ff7a8133500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.744 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.740+0000 7ff7a7fff640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff7a806e2d0 0x7ff7a8132fc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:34824/0 (socket says 172.21.15.92:34824) 2024-09-21T09:43:04.744 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.740+0000 7ff7a7fff640 1 -- 172.21.15.92:0/1328699431 learned_addr learned my addr 172.21.15.92:0/1328699431 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:04.744 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.740+0000 7ff7a77fe640 1 -- 172.21.15.92:0/1328699431 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff7a806e2d0 msgr2=0x7ff7a8132fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.744 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.740+0000 7ff7acae6640 1 --2- 172.21.15.92:0/1328699431 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff7a8091510 0x7ff7a8133a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.744 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.741+0000 7ff7a77fe640 1 --2- 172.21.15.92:0/1328699431 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff7a806e2d0 0x7ff7a8132fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.744 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.741+0000 7ff7a77fe640 1 -- 172.21.15.92:0/1328699431 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff7a8091510 msgr2=0x7ff7a8133a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.744 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.741+0000 7ff7a77fe640 1 --2- 172.21.15.92:0/1328699431 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff7a8091510 0x7ff7a8133a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.744 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.741+0000 7ff7a77fe640 1 -- 172.21.15.92:0/1328699431 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff7a81260c0 con 0x7ff7a806ebf0 2024-09-21T09:43:04.744 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.741+0000 7ff7acae6640 1 --2- 172.21.15.92:0/1328699431 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff7a8091510 0x7ff7a8133a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T09:43:04.744 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.741+0000 7ff7a7fff640 1 --2- 172.21.15.92:0/1328699431 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff7a806e2d0 0x7ff7a8132fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:04.745 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.741+0000 7ff7a77fe640 1 --2- 172.21.15.92:0/1328699431 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff7a806ebf0 0x7ff7a8133500 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7ff79c00e9f0 tx=0x7ff79c00eeb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:04.745 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.741+0000 7ff7a4ff9640 1 -- 172.21.15.92:0/1328699431 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff79c00ccc0 con 0x7ff7a806ebf0 2024-09-21T09:43:04.745 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.741+0000 7ff7a4ff9640 1 -- 172.21.15.92:0/1328699431 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7ff79c00ce60 con 0x7ff7a806ebf0 2024-09-21T09:43:04.745 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.741+0000 7ff7ae549640 1 -- 172.21.15.92:0/1328699431 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff7a81264d0 con 0x7ff7a806ebf0 2024-09-21T09:43:04.745 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.741+0000 7ff7a4ff9640 1 -- 172.21.15.92:0/1328699431 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff79c004290 con 0x7ff7a806ebf0 2024-09-21T09:43:04.745 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.741+0000 7ff7ae549640 1 -- 172.21.15.92:0/1328699431 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7ff7a808d320 con 0x7ff7a806ebf0 2024-09-21T09:43:04.745 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.743+0000 7ff78e7fc640 1 -- 172.21.15.92:0/1328699431 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff7a80008d0 con 0x7ff7a806ebf0 2024-09-21T09:43:04.746 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.744+0000 7ff7a4ff9640 1 -- 172.21.15.92:0/1328699431 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7ff79c0035f0 con 0x7ff7a806ebf0 2024-09-21T09:43:04.746 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.744+0000 7ff7a7fff640 1 -- 172.21.15.92:0/1328699431 >> v1:172.21.15.92:6800/1280404317 conn(0x7ff794047fd0 legacy=0x7ff79404a490 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.92:6800/1280404317 2024-09-21T09:43:04.746 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.745+0000 7ff7a4ff9640 1 -- 172.21.15.92:0/1328699431 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7ff79c05e830 con 0x7ff7a806ebf0 2024-09-21T09:43:04.752 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.751+0000 7ff7a4ff9640 1 -- 172.21.15.92:0/1328699431 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+152684 (secure 0 0 0) 0x7ff79c004430 con 0x7ff7a806ebf0 2024-09-21T09:43:04.822 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.819+0000 7fec06f40640 1 -- 172.21.15.92:0/1833344899 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fec00098d40 msgr2=0x7fec00099120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.822 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.819+0000 7fec06f40640 1 --2- 172.21.15.92:0/1833344899 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fec00098d40 0x7fec00099120 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7febf4009880 tx=0x7febf402a440 comp rx=0 tx=0).stop 2024-09-21T09:43:04.822 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.820+0000 7fec06f40640 1 -- 172.21.15.92:0/1833344899 shutdown_connections 2024-09-21T09:43:04.822 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.820+0000 7fec06f40640 1 --2- 172.21.15.92:0/1833344899 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fec00099660 0x7fec0012eab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.822 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.820+0000 7fec06f40640 1 --2- 172.21.15.92:0/1833344899 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fec00098d40 0x7fec00099120 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.822 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.820+0000 7fec06f40640 1 -- 172.21.15.92:0/1833344899 >> 172.21.15.92:0/1833344899 conn(0x7fec000a1790 msgr2=0x7fec000a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:04.822 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.820+0000 7fec06f40640 1 -- 172.21.15.92:0/1833344899 shutdown_connections 2024-09-21T09:43:04.822 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.820+0000 7fec06f40640 1 -- 172.21.15.92:0/1833344899 wait complete. 2024-09-21T09:43:04.823 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.821+0000 7fec06f40640 1 Processor -- start 2024-09-21T09:43:04.823 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.821+0000 7fec06f40640 1 -- start start 2024-09-21T09:43:04.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.822+0000 7fec06f40640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fec00098d40 0x7fec001c4f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.822+0000 7fec06f40640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fec00099660 0x7fec001c54c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.822+0000 7fec04cdc640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fec00098d40 0x7fec001c4f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.822+0000 7fec04cdc640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fec00098d40 0x7fec001c4f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:34836/0 (socket says 172.21.15.92:34836) 2024-09-21T09:43:04.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.822+0000 7fec04cdc640 1 -- 172.21.15.92:0/1175666627 learned_addr learned my addr 172.21.15.92:0/1175666627 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:04.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.823+0000 7febfffff640 1 --2- 172.21.15.92:0/1175666627 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fec00099660 0x7fec001c54c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.823+0000 7fec06f40640 1 --2- 172.21.15.92:0/1175666627 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fec0012eff0 0x7fec00071310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:04.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.823+0000 7fec06f40640 1 -- 172.21.15.92:0/1175666627 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fec001378c0 con 0x7fec0012eff0 2024-09-21T09:43:04.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.823+0000 7fec06f40640 1 -- 172.21.15.92:0/1175666627 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fec00137740 con 0x7fec00098d40 2024-09-21T09:43:04.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.823+0000 7fec06f40640 1 -- 172.21.15.92:0/1175666627 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fec00137a40 con 0x7fec00099660 2024-09-21T09:43:04.826 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.823+0000 7fec054dd640 1 --2- 172.21.15.92:0/1175666627 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fec0012eff0 0x7fec00071310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:04.826 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.823+0000 7fec054dd640 1 -- 172.21.15.92:0/1175666627 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fec00099660 msgr2=0x7fec001c54c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.823+0000 7fec054dd640 1 --2- 172.21.15.92:0/1175666627 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fec00099660 0x7fec001c54c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.823+0000 7fec054dd640 1 -- 172.21.15.92:0/1175666627 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fec00098d40 msgr2=0x7fec001c4f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.824+0000 7fec054dd640 1 --2- 172.21.15.92:0/1175666627 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fec00098d40 0x7fec001c4f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.824+0000 7fec054dd640 1 -- 172.21.15.92:0/1175666627 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fec00073380 con 0x7fec0012eff0 2024-09-21T09:43:04.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.824+0000 7fec04cdc640 1 --2- 172.21.15.92:0/1175666627 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fec00098d40 0x7fec001c4f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:04.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.824+0000 7fec054dd640 1 --2- 172.21.15.92:0/1175666627 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fec0012eff0 0x7fec00071310 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7febf800c8e0 tx=0x7febf800cda0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:04.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.824+0000 7febfd7fa640 1 -- 172.21.15.92:0/1175666627 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7febf8003b70 con 0x7fec0012eff0 2024-09-21T09:43:04.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.824+0000 7febfd7fa640 1 -- 172.21.15.92:0/1175666627 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7febf8003d10 con 0x7fec0012eff0 2024-09-21T09:43:04.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.824+0000 7febfd7fa640 1 -- 172.21.15.92:0/1175666627 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7febf80052d0 con 0x7fec0012eff0 2024-09-21T09:43:04.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.824+0000 7fec06f40640 1 -- 172.21.15.92:0/1175666627 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fec00073d80 con 0x7fec0012eff0 2024-09-21T09:43:04.828 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.825+0000 7fec06f40640 1 -- 172.21.15.92:0/1175666627 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fec001c5a00 con 0x7fec0012eff0 2024-09-21T09:43:04.828 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.826+0000 7febfd7fa640 1 -- 172.21.15.92:0/1175666627 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7febf8005470 con 0x7fec0012eff0 2024-09-21T09:43:04.828 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.827+0000 7fec04cdc640 1 -- 172.21.15.92:0/1175666627 >> v1:172.21.15.92:6800/1280404317 conn(0x7febe4048020 legacy=0x7febe404a4e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.92:6800/1280404317 2024-09-21T09:43:04.829 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.827+0000 7febfd7fa640 1 -- 172.21.15.92:0/1175666627 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7febf805e580 con 0x7fec0012eff0 2024-09-21T09:43:04.829 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.827+0000 7fec06f40640 1 -- 172.21.15.92:0/1175666627 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fec00099120 con 0x7fec0012eff0 2024-09-21T09:43:04.836 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.834+0000 7febfd7fa640 1 -- 172.21.15.92:0/1175666627 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+152684 (secure 0 0 0) 0x7fec00073d80 con 0x7fec0012eff0 2024-09-21T09:43:04.895 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.893+0000 7f066bfff640 1 -- 172.21.15.92:0/1140450962 >> v1:172.21.15.92:6800/1280404317 conn(0x7f0650047fd0 legacy=0x7f065004a490 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.92:6800/1280404317 2024-09-21T09:43:04.897 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.896+0000 7fab5b628640 1 -- 172.21.15.92:0/772643251 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7fab54125300 con 0x7fab54129510 2024-09-21T09:43:04.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.896+0000 7fab49ffb640 1 -- 172.21.15.92:0/772643251 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fab5003c3c0 con 0x7fab54129510 2024-09-21T09:43:04.898 INFO:teuthology.orchestra.run.smithi092.stdout:47244640275 2024-09-21T09:43:04.900 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.898+0000 7fab5b628640 1 -- 172.21.15.92:0/772643251 >> v1:172.21.15.92:6800/1280404317 conn(0x7fab3804c3e0 legacy=0x7fab3804e8a0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:43:04.901 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.898+0000 7fab5b628640 1 -- 172.21.15.92:0/772643251 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fab54129510 msgr2=0x7fab54130610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.901 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.898+0000 7fab5b628640 1 --2- 172.21.15.92:0/772643251 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fab54129510 0x7fab54130610 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7fab5000d950 tx=0x7fab5000de10 comp rx=0 tx=0).stop 2024-09-21T09:43:04.901 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.898+0000 7fab5b628640 1 -- 172.21.15.92:0/772643251 shutdown_connections 2024-09-21T09:43:04.901 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.898+0000 7fab5b628640 1 --2- 172.21.15.92:0/772643251 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fab5412cd60 0x7fab54131090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.901 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.898+0000 7fab5b628640 1 --2- 172.21.15.92:0/772643251 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fab54129e30 0x7fab54130b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.901 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.898+0000 7fab5b628640 1 --2- 172.21.15.92:0/772643251 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fab54129510 0x7fab54130610 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.901 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.898+0000 7fab5b628640 1 -- 172.21.15.92:0/772643251 >> 172.21.15.92:0/772643251 conn(0x7fab5409d370 msgr2=0x7fab54134b20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:04.901 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.899+0000 7fab5b628640 1 -- 172.21.15.92:0/772643251 shutdown_connections 2024-09-21T09:43:04.901 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.899+0000 7fab5b628640 1 -- 172.21.15.92:0/772643251 wait complete. 2024-09-21T09:43:04.911 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640276 got 47244640275 for osd.0 2024-09-21T09:43:04.948 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.946+0000 7ff7a7fff640 1 -- 172.21.15.92:0/1328699431 >> v1:172.21.15.92:6800/1280404317 conn(0x7ff794047fd0 legacy=0x7ff79404a490 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.92:6800/1280404317 2024-09-21T09:43:04.977 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.975+0000 7f0656ffd640 1 -- 172.21.15.92:0/1140450962 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7f066c0008d0 con 0x7f066c0916e0 2024-09-21T09:43:04.977 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.975+0000 7f06697fa640 1 -- 172.21.15.92:0/1140450962 <== mon.1 v2:172.21.15.92:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f066003f180 con 0x7f066c0916e0 2024-09-21T09:43:04.977 INFO:teuthology.orchestra.run.smithi092.stdout:47244640275 2024-09-21T09:43:04.981 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.978+0000 7f0656ffd640 1 -- 172.21.15.92:0/1140450962 >> v1:172.21.15.92:6800/1280404317 conn(0x7f0650047fd0 legacy=0x7f065004a490 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:43:04.981 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.979+0000 7f0656ffd640 1 -- 172.21.15.92:0/1140450962 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f066c0916e0 msgr2=0x7f066c06f710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:04.981 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.979+0000 7f0656ffd640 1 --2- 172.21.15.92:0/1140450962 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f066c0916e0 0x7f066c06f710 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f066000d950 tx=0x7f066000de10 comp rx=0 tx=0).stop 2024-09-21T09:43:04.981 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.979+0000 7f0656ffd640 1 -- 172.21.15.92:0/1140450962 shutdown_connections 2024-09-21T09:43:04.981 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.979+0000 7f0656ffd640 1 --2- 172.21.15.92:0/1140450962 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f066c12d2a0 0x7f066c070190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.981 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.979+0000 7f0656ffd640 1 --2- 172.21.15.92:0/1140450962 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f066c12c980 0x7f066c06fc50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.981 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.979+0000 7f0656ffd640 1 --2- 172.21.15.92:0/1140450962 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f066c0916e0 0x7f066c06f710 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:04.981 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.979+0000 7f0656ffd640 1 -- 172.21.15.92:0/1140450962 >> 172.21.15.92:0/1140450962 conn(0x7f066c09c470 msgr2=0x7f066c0939b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:04.981 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.979+0000 7f0656ffd640 1 -- 172.21.15.92:0/1140450962 shutdown_connections 2024-09-21T09:43:04.981 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:04.979+0000 7f0656ffd640 1 -- 172.21.15.92:0/1140450962 wait complete. 2024-09-21T09:43:04.996 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640276 got 47244640275 for osd.2 2024-09-21T09:43:05.030 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.028+0000 7fec04cdc640 1 -- 172.21.15.92:0/1175666627 >> v1:172.21.15.92:6800/1280404317 conn(0x7febe4048020 legacy=0x7febe404a4e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.92:6800/1280404317 2024-09-21T09:43:05.032 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.031+0000 7ff78e7fc640 1 -- 172.21.15.92:0/1328699431 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7ff7a8090ea0 con 0x7ff7a806ebf0 2024-09-21T09:43:05.033 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.031+0000 7ff7a4ff9640 1 -- 172.21.15.92:0/1328699431 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7ff79c03da80 con 0x7ff7a806ebf0 2024-09-21T09:43:05.033 INFO:teuthology.orchestra.run.smithi092.stdout:47244640273 2024-09-21T09:43:05.035 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.033+0000 7ff78e7fc640 1 -- 172.21.15.92:0/1328699431 >> v1:172.21.15.92:6800/1280404317 conn(0x7ff794047fd0 legacy=0x7ff79404a490 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:43:05.035 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.033+0000 7ff78e7fc640 1 -- 172.21.15.92:0/1328699431 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff7a806ebf0 msgr2=0x7ff7a8133500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:05.035 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.033+0000 7ff78e7fc640 1 --2- 172.21.15.92:0/1328699431 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff7a806ebf0 0x7ff7a8133500 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7ff79c00e9f0 tx=0x7ff79c00eeb0 comp rx=0 tx=0).stop 2024-09-21T09:43:05.035 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.033+0000 7ff78e7fc640 1 -- 172.21.15.92:0/1328699431 shutdown_connections 2024-09-21T09:43:05.035 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.033+0000 7ff78e7fc640 1 --2- 172.21.15.92:0/1328699431 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff7a8091510 0x7ff7a8133a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:05.035 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.033+0000 7ff78e7fc640 1 --2- 172.21.15.92:0/1328699431 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff7a806ebf0 0x7ff7a8133500 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:05.035 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.033+0000 7ff78e7fc640 1 --2- 172.21.15.92:0/1328699431 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff7a806e2d0 0x7ff7a8132fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:05.035 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.033+0000 7ff78e7fc640 1 -- 172.21.15.92:0/1328699431 >> 172.21.15.92:0/1328699431 conn(0x7ff7a809c230 msgr2=0x7ff7a80938d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:05.035 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.033+0000 7ff78e7fc640 1 -- 172.21.15.92:0/1328699431 shutdown_connections 2024-09-21T09:43:05.036 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.033+0000 7ff78e7fc640 1 -- 172.21.15.92:0/1328699431 wait complete. 2024-09-21T09:43:05.046 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.044+0000 7fec06f40640 1 -- 172.21.15.92:0/1175666627 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7fec00074060 con 0x7fec0012eff0 2024-09-21T09:43:05.047 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640274 got 47244640273 for osd.1 2024-09-21T09:43:05.047 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.045+0000 7febfd7fa640 1 -- 172.21.15.92:0/1175666627 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7febf803c120 con 0x7fec0012eff0 2024-09-21T09:43:05.048 INFO:teuthology.orchestra.run.smithi092.stdout:38654705682 2024-09-21T09:43:05.049 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.047+0000 7fec06f40640 1 -- 172.21.15.92:0/1175666627 >> v1:172.21.15.92:6800/1280404317 conn(0x7febe4048020 legacy=0x7febe404a4e0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:43:05.049 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.047+0000 7fec06f40640 1 -- 172.21.15.92:0/1175666627 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fec0012eff0 msgr2=0x7fec00071310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:05.049 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.047+0000 7fec06f40640 1 --2- 172.21.15.92:0/1175666627 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fec0012eff0 0x7fec00071310 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7febf800c8e0 tx=0x7febf800cda0 comp rx=0 tx=0).stop 2024-09-21T09:43:05.049 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.047+0000 7fec06f40640 1 -- 172.21.15.92:0/1175666627 shutdown_connections 2024-09-21T09:43:05.049 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.047+0000 7fec06f40640 1 --2- 172.21.15.92:0/1175666627 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fec0012eff0 0x7fec00071310 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:05.049 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.047+0000 7fec06f40640 1 --2- 172.21.15.92:0/1175666627 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fec00099660 0x7fec001c54c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:05.049 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.047+0000 7fec06f40640 1 --2- 172.21.15.92:0/1175666627 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fec00098d40 0x7fec001c4f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:05.049 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.047+0000 7fec06f40640 1 -- 172.21.15.92:0/1175666627 >> 172.21.15.92:0/1175666627 conn(0x7fec000a1790 msgr2=0x7fec0009cbe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:05.049 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.047+0000 7fec06f40640 1 -- 172.21.15.92:0/1175666627 shutdown_connections 2024-09-21T09:43:05.049 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:05.047+0000 7fec06f40640 1 -- 172.21.15.92:0/1175666627 wait complete. 2024-09-21T09:43:05.060 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705682 for osd.3 2024-09-21T09:43:05.913 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T09:43:05.997 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T09:43:06.020 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.015+0000 7fe687a57640 1 -- 172.21.15.92:0/1811596641 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe680098d40 msgr2=0x7fe680099120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:06.020 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.015+0000 7fe687a57640 1 --2- 172.21.15.92:0/1811596641 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe680098d40 0x7fe680099120 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7fe67c009860 tx=0x7fe67c02a3d0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.020 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.015+0000 7fe687a57640 1 -- 172.21.15.92:0/1811596641 shutdown_connections 2024-09-21T09:43:06.020 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.015+0000 7fe687a57640 1 --2- 172.21.15.92:0/1811596641 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe68012f1c0 0x7fe68012f650 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.021 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.015+0000 7fe687a57640 1 --2- 172.21.15.92:0/1811596641 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe680098d40 0x7fe680099120 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.021 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.015+0000 7fe687a57640 1 -- 172.21.15.92:0/1811596641 >> 172.21.15.92:0/1811596641 conn(0x7fe6800a1790 msgr2=0x7fe6800a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:06.021 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.015+0000 7fe687a57640 1 -- 172.21.15.92:0/1811596641 shutdown_connections 2024-09-21T09:43:06.021 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.015+0000 7fe687a57640 1 -- 172.21.15.92:0/1811596641 wait complete. 2024-09-21T09:43:06.021 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.016+0000 7fe687a57640 1 Processor -- start 2024-09-21T09:43:06.021 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.016+0000 7fe687a57640 1 -- start start 2024-09-21T09:43:06.021 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.016+0000 7fe687a57640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe680098d40 0x7fe6801c4fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:06.021 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.017+0000 7fe687a57640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe680099660 0x7fe6801c5510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:06.021 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.017+0000 7fe687a57640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe68012f1c0 0x7fe680071530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:06.021 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.017+0000 7fe687a57640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fe680137910 con 0x7fe680098d40 2024-09-21T09:43:06.021 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.017+0000 7fe687a57640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fe680137790 con 0x7fe68012f1c0 2024-09-21T09:43:06.021 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.017+0000 7fe687a57640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fe680137a90 con 0x7fe680099660 2024-09-21T09:43:06.021 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.017+0000 7fe6857f3640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe680098d40 0x7fe6801c4fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:06.021 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.017+0000 7fe6857f3640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe680098d40 0x7fe6801c4fd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:48434/0 (socket says 172.21.15.92:48434) 2024-09-21T09:43:06.022 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.017+0000 7fe6857f3640 1 -- 172.21.15.92:0/2263859485 learned_addr learned my addr 172.21.15.92:0/2263859485 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:06.022 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.017+0000 7fe684ff2640 1 --2- 172.21.15.92:0/2263859485 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe680099660 0x7fe6801c5510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:06.022 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.017+0000 7fe6857f3640 1 -- 172.21.15.92:0/2263859485 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe680099660 msgr2=0x7fe6801c5510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:06.022 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.017+0000 7fe6857f3640 1 --2- 172.21.15.92:0/2263859485 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe680099660 0x7fe6801c5510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.022 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.017+0000 7fe6857f3640 1 -- 172.21.15.92:0/2263859485 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe68012f1c0 msgr2=0x7fe680071530 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:06.022 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.017+0000 7fe685ff4640 1 --2- 172.21.15.92:0/2263859485 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe68012f1c0 0x7fe680071530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:06.022 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.017+0000 7fe6857f3640 1 --2- 172.21.15.92:0/2263859485 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe68012f1c0 0x7fe680071530 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.022 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.017+0000 7fe6857f3640 1 -- 172.21.15.92:0/2263859485 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe6801c5a50 con 0x7fe680098d40 2024-09-21T09:43:06.022 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.017+0000 7fe684ff2640 1 --2- 172.21.15.92:0/2263859485 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe680099660 0x7fe6801c5510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:06.022 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.017+0000 7fe685ff4640 1 --2- 172.21.15.92:0/2263859485 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe68012f1c0 0x7fe680071530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T09:43:06.022 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.017+0000 7fe6857f3640 1 --2- 172.21.15.92:0/2263859485 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe680098d40 0x7fe6801c4fd0 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7fe67c009780 tx=0x7fe67c033ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:06.022 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.018+0000 7fe675ffb640 1 -- 172.21.15.92:0/2263859485 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe67c031b50 con 0x7fe680098d40 2024-09-21T09:43:06.023 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.018+0000 7fe675ffb640 1 -- 172.21.15.92:0/2263859485 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fe67c031e00 con 0x7fe680098d40 2024-09-21T09:43:06.023 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.018+0000 7fe675ffb640 1 -- 172.21.15.92:0/2263859485 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe67c005280 con 0x7fe680098d40 2024-09-21T09:43:06.023 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.018+0000 7fe687a57640 1 -- 172.21.15.92:0/2263859485 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe680073fa0 con 0x7fe680098d40 2024-09-21T09:43:06.023 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.019+0000 7fe687a57640 1 -- 172.21.15.92:0/2263859485 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe680071c50 con 0x7fe680098d40 2024-09-21T09:43:06.023 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.020+0000 7fe687a57640 1 -- 172.21.15.92:0/2263859485 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe68012ca30 con 0x7fe680098d40 2024-09-21T09:43:06.024 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.020+0000 7fe675ffb640 1 -- 172.21.15.92:0/2263859485 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fe67c005420 con 0x7fe680098d40 2024-09-21T09:43:06.024 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.020+0000 7fe675ffb640 1 -- 172.21.15.92:0/2263859485 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7fe67c07e7a0 con 0x7fe680098d40 2024-09-21T09:43:06.024 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.020+0000 7fe684ff2640 1 -- 172.21.15.92:0/2263859485 >> v1:172.21.15.92:6800/1280404317 conn(0x7fe664048020 legacy=0x7fe66404a4e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.92:6800/1280404317 2024-09-21T09:43:06.029 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.027+0000 7fe675ffb640 1 -- 172.21.15.92:0/2263859485 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+152684 (secure 0 0 0) 0x7fe68012ca30 con 0x7fe680098d40 2024-09-21T09:43:06.048 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T09:43:06.061 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T09:43:06.107 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.105+0000 7fd49903c640 1 -- 172.21.15.92:0/2590738769 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd494074200 msgr2=0x7fd49406f4d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:06.107 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.105+0000 7fd49903c640 1 --2- 172.21.15.92:0/2590738769 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd494074200 0x7fd49406f4d0 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7fd488005fd0 tx=0x7fd48802a420 comp rx=0 tx=0).stop 2024-09-21T09:43:06.107 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.105+0000 7fd49903c640 1 -- 172.21.15.92:0/2590738769 shutdown_connections 2024-09-21T09:43:06.107 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.105+0000 7fd49903c640 1 --2- 172.21.15.92:0/2590738769 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd494074200 0x7fd49406f4d0 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.107 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.105+0000 7fd49903c640 1 --2- 172.21.15.92:0/2590738769 >> v2:172.21.15.92:3300/0 conn(0x7fd494073e20 0x7fd494076360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.107 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.105+0000 7fd49903c640 1 -- 172.21.15.92:0/2590738769 >> 172.21.15.92:0/2590738769 conn(0x7fd49409dde0 msgr2=0x7fd49409e1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:06.107 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.105+0000 7fd49903c640 1 -- 172.21.15.92:0/2590738769 shutdown_connections 2024-09-21T09:43:06.107 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.105+0000 7fd49903c640 1 -- 172.21.15.92:0/2590738769 wait complete. 2024-09-21T09:43:06.109 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.107+0000 7fd49903c640 1 Processor -- start 2024-09-21T09:43:06.109 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.107+0000 7fd49903c640 1 -- start start 2024-09-21T09:43:06.109 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.108+0000 7fd49903c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd49406fa10 0x7fd494094440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:06.110 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.108+0000 7fd492d9d640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd49406fa10 0x7fd494094440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:06.110 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.108+0000 7fd492d9d640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd49406fa10 0x7fd494094440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:34878/0 (socket says 172.21.15.92:34878) 2024-09-21T09:43:06.110 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.108+0000 7fd492d9d640 1 -- 172.21.15.92:0/1331720353 learned_addr learned my addr 172.21.15.92:0/1331720353 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:06.111 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.108+0000 7fd49903c640 1 --2- 172.21.15.92:0/1331720353 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd494073e20 0x7fd494094980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:06.111 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.108+0000 7fd49903c640 1 --2- 172.21.15.92:0/1331720353 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd494074200 0x7fd494094ec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:06.111 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.108+0000 7fd49259c640 1 --2- 172.21.15.92:0/1331720353 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd494073e20 0x7fd494094980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:06.111 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.109+0000 7fd49903c640 1 -- 172.21.15.92:0/1331720353 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fd4941377a0 con 0x7fd494073e20 2024-09-21T09:43:06.111 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.109+0000 7fd49903c640 1 -- 172.21.15.92:0/1331720353 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fd494137620 con 0x7fd49406fa10 2024-09-21T09:43:06.111 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.109+0000 7fd49903c640 1 -- 172.21.15.92:0/1331720353 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fd494137920 con 0x7fd494074200 2024-09-21T09:43:06.111 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.109+0000 7fd49359e640 1 --2- 172.21.15.92:0/1331720353 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd494074200 0x7fd494094ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:06.112 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.109+0000 7fd49359e640 1 -- 172.21.15.92:0/1331720353 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd49406fa10 msgr2=0x7fd494094440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:06.112 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.109+0000 7fd49359e640 1 --2- 172.21.15.92:0/1331720353 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd49406fa10 0x7fd494094440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.112 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.109+0000 7fd49359e640 1 -- 172.21.15.92:0/1331720353 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd494073e20 msgr2=0x7fd494094980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:06.112 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.109+0000 7fd49359e640 1 --2- 172.21.15.92:0/1331720353 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd494073e20 0x7fd494094980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.112 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.109+0000 7fd49359e640 1 -- 172.21.15.92:0/1331720353 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd494131b80 con 0x7fd494074200 2024-09-21T09:43:06.112 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.109+0000 7fd492d9d640 1 --2- 172.21.15.92:0/1331720353 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd49406fa10 0x7fd494094440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:06.113 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.110+0000 7fd49359e640 1 --2- 172.21.15.92:0/1331720353 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd494074200 0x7fd494094ec0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7fd48c00c780 tx=0x7fd48c00cc40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:06.113 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.110+0000 7fd47b7fe640 1 -- 172.21.15.92:0/1331720353 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd48c016020 con 0x7fd494074200 2024-09-21T09:43:06.113 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.110+0000 7fd47b7fe640 1 -- 172.21.15.92:0/1331720353 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fd48c00ab40 con 0x7fd494074200 2024-09-21T09:43:06.113 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.110+0000 7fd47b7fe640 1 -- 172.21.15.92:0/1331720353 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd48c00ce30 con 0x7fd494074200 2024-09-21T09:43:06.113 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.111+0000 7fd49903c640 1 -- 172.21.15.92:0/1331720353 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd494122a80 con 0x7fd494074200 2024-09-21T09:43:06.113 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.112+0000 7fd49903c640 1 -- 172.21.15.92:0/1331720353 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fd494096180 con 0x7fd494074200 2024-09-21T09:43:06.115 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.113+0000 7fd49903c640 1 -- 172.21.15.92:0/1331720353 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd494135520 con 0x7fd494074200 2024-09-21T09:43:06.115 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.113+0000 7fd47b7fe640 1 -- 172.21.15.92:0/1331720353 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fd48c019020 con 0x7fd494074200 2024-09-21T09:43:06.115 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.114+0000 7fd492d9d640 1 -- 172.21.15.92:0/1331720353 >> v1:172.21.15.92:6800/1280404317 conn(0x7fd480047fd0 legacy=0x7fd48004a490 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.92:6800/1280404317 2024-09-21T09:43:06.116 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.114+0000 7fd47b7fe640 1 -- 172.21.15.92:0/1331720353 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7fd48c05e540 con 0x7fd494074200 2024-09-21T09:43:06.122 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.121+0000 7fd47b7fe640 1 -- 172.21.15.92:0/1331720353 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+152684 (secure 0 0 0) 0x7fd48c065070 con 0x7fd494074200 2024-09-21T09:43:06.181 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.180+0000 7fbead59a640 1 -- 172.21.15.92:0/391491744 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbea00047a0 con 0x7fbeb006e2d0 2024-09-21T09:43:06.182 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.181+0000 7fbeb4c4b640 1 -- 172.21.15.92:0/391491744 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbeb006e2d0 msgr2=0x7fbeb006e6b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:06.182 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.181+0000 7fbeb4c4b640 1 --2- 172.21.15.92:0/391491744 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbeb006e2d0 0x7fbeb006e6b0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fbeb0094f90 tx=0x7fbea002a4b0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.183 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.181+0000 7fbeb4c4b640 1 -- 172.21.15.92:0/391491744 shutdown_connections 2024-09-21T09:43:06.183 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.181+0000 7fbeb4c4b640 1 --2- 172.21.15.92:0/391491744 >> v2:172.21.15.92:3300/0 conn(0x7fbeb006ebf0 0x7fbeb0090fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.183 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.181+0000 7fbeb4c4b640 1 --2- 172.21.15.92:0/391491744 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbeb006e2d0 0x7fbeb006e6b0 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.183 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.181+0000 7fbeb4c4b640 1 -- 172.21.15.92:0/391491744 >> 172.21.15.92:0/391491744 conn(0x7fbeb009e490 msgr2=0x7fbeb009e8a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:06.183 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.181+0000 7fbeb4c4b640 1 -- 172.21.15.92:0/391491744 shutdown_connections 2024-09-21T09:43:06.183 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.181+0000 7fbeb4c4b640 1 -- 172.21.15.92:0/391491744 wait complete. 2024-09-21T09:43:06.184 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.183+0000 7fbeb4c4b640 1 Processor -- start 2024-09-21T09:43:06.184 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.183+0000 7fbeb4c4b640 1 -- start start 2024-09-21T09:43:06.186 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.184+0000 7fbeb4c4b640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbeb006ebf0 0x7fbeb006f710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:06.186 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.184+0000 7fbeb4c4b640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbeb0091510 0x7fbeb006fc50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:06.186 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.184+0000 7fbeb4c4b640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbeb0071480 0x7fbeb0070190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:06.187 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.184+0000 7fbeae59c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbeb006ebf0 0x7fbeb006f710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:06.187 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.184+0000 7fbeae59c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbeb006ebf0 0x7fbeb006f710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:53318/0 (socket says 172.21.15.92:53318) 2024-09-21T09:43:06.187 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.184+0000 7fbeaed9d640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbeb0071480 0x7fbeb0070190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:06.187 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.184+0000 7fbeadd9b640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbeb0091510 0x7fbeb006fc50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:06.187 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.184+0000 7fbeaed9d640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbeb0071480 0x7fbeb0070190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:49102/0 (socket says 172.21.15.92:49102) 2024-09-21T09:43:06.187 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.184+0000 7fbeadd9b640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbeb0091510 0x7fbeb006fc50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:59366/0 (socket says 172.21.15.92:59366) 2024-09-21T09:43:06.187 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.184+0000 7fbeae59c640 1 -- 172.21.15.92:0/3925994257 learned_addr learned my addr 172.21.15.92:0/3925994257 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:06.187 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.184+0000 7fbeb4c4b640 1 -- 172.21.15.92:0/3925994257 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fbeb0135860 con 0x7fbeb006ebf0 2024-09-21T09:43:06.187 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.185+0000 7fbeb4c4b640 1 -- 172.21.15.92:0/3925994257 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fbeb01356e0 con 0x7fbeb0091510 2024-09-21T09:43:06.187 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.185+0000 7fbeb4c4b640 1 -- 172.21.15.92:0/3925994257 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fbeb01359e0 con 0x7fbeb0071480 2024-09-21T09:43:06.188 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.185+0000 7fbeaed9d640 1 -- 172.21.15.92:0/3925994257 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbeb0091510 msgr2=0x7fbeb006fc50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:06.188 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.185+0000 7fbeaed9d640 1 --2- 172.21.15.92:0/3925994257 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbeb0091510 0x7fbeb006fc50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.188 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.185+0000 7fbeaed9d640 1 -- 172.21.15.92:0/3925994257 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbeb006ebf0 msgr2=0x7fbeb006f710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:06.188 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.185+0000 7fbeaed9d640 1 --2- 172.21.15.92:0/3925994257 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbeb006ebf0 0x7fbeb006f710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.188 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.185+0000 7fbeaed9d640 1 -- 172.21.15.92:0/3925994257 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbeb0070af0 con 0x7fbeb0071480 2024-09-21T09:43:06.189 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.185+0000 7fbeae59c640 1 --2- 172.21.15.92:0/3925994257 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbeb006ebf0 0x7fbeb006f710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:06.189 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.185+0000 7fbeadd9b640 1 --2- 172.21.15.92:0/3925994257 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbeb0091510 0x7fbeb006fc50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:43:06.189 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.186+0000 7fbeaed9d640 1 --2- 172.21.15.92:0/3925994257 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbeb0071480 0x7fbeb0070190 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7fbea800c670 tx=0x7fbea800cb30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:06.189 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.186+0000 7fbe9effd640 1 -- 172.21.15.92:0/3925994257 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbea8017070 con 0x7fbeb0071480 2024-09-21T09:43:06.189 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.186+0000 7fbe9effd640 1 -- 172.21.15.92:0/3925994257 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fbea8003c80 con 0x7fbeb0071480 2024-09-21T09:43:06.189 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.186+0000 7fbe9effd640 1 -- 172.21.15.92:0/3925994257 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbea80052d0 con 0x7fbeb0071480 2024-09-21T09:43:06.189 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.187+0000 7fbeb4c4b640 1 -- 172.21.15.92:0/3925994257 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbeb0126850 con 0x7fbeb0071480 2024-09-21T09:43:06.190 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.189+0000 7fbeb4c4b640 1 -- 172.21.15.92:0/3925994257 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fbeb0092580 con 0x7fbeb0071480 2024-09-21T09:43:06.191 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.189+0000 7fbe9effd640 1 -- 172.21.15.92:0/3925994257 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7fbea801a070 con 0x7fbeb0071480 2024-09-21T09:43:06.191 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.190+0000 7fbe9effd640 1 -- 172.21.15.92:0/3925994257 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7fbea80622e0 con 0x7fbeb0071480 2024-09-21T09:43:06.192 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.190+0000 7fbeae59c640 1 -- 172.21.15.92:0/3925994257 >> v1:172.21.15.92:6800/1280404317 conn(0x7fbe90048020 legacy=0x7fbe9004a4e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.92:6800/1280404317 2024-09-21T09:43:06.192 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.190+0000 7fbe9cff9640 1 -- 172.21.15.92:0/3925994257 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbeb006e6b0 con 0x7fbeb0071480 2024-09-21T09:43:06.196 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.194+0000 7fbe9effd640 1 -- 172.21.15.92:0/3925994257 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+152684 (secure 0 0 0) 0x7fbea803be80 con 0x7fbeb0071480 2024-09-21T09:43:06.208 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.206+0000 7f254e174640 1 -- 172.21.15.92:0/155013495 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2548098d40 msgr2=0x7f2548099120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:06.208 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.206+0000 7f254e174640 1 --2- 172.21.15.92:0/155013495 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2548098d40 0x7f2548099120 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f2538009830 tx=0x7f253802a3f0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.208 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.206+0000 7f254e174640 1 -- 172.21.15.92:0/155013495 shutdown_connections 2024-09-21T09:43:06.208 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.206+0000 7f254e174640 1 --2- 172.21.15.92:0/155013495 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f254812f1c0 0x7f254812f600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.208 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.206+0000 7f254e174640 1 --2- 172.21.15.92:0/155013495 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2548098d40 0x7f2548099120 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.208 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.206+0000 7f254e174640 1 -- 172.21.15.92:0/155013495 >> 172.21.15.92:0/155013495 conn(0x7f25480a1790 msgr2=0x7f25480a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:06.208 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.206+0000 7f254e174640 1 -- 172.21.15.92:0/155013495 shutdown_connections 2024-09-21T09:43:06.208 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.206+0000 7f254e174640 1 -- 172.21.15.92:0/155013495 wait complete. 2024-09-21T09:43:06.211 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.207+0000 7f254e174640 1 Processor -- start 2024-09-21T09:43:06.211 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.207+0000 7f254e174640 1 -- start start 2024-09-21T09:43:06.211 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.207+0000 7f254e174640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2548098d40 0x7f25480714c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:06.211 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.207+0000 7f254e174640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2548099660 0x7f2548071a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:06.211 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.207+0000 7f254e174640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f254812f1c0 0x7f2548071f40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:06.211 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.207+0000 7f25477fe640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2548098d40 0x7f25480714c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:06.211 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.208+0000 7f25477fe640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2548098d40 0x7f25480714c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:49104/0 (socket says 172.21.15.92:49104) 2024-09-21T09:43:06.211 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.208+0000 7f25477fe640 1 -- 172.21.15.92:0/917449321 learned_addr learned my addr 172.21.15.92:0/917449321 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:06.211 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.208+0000 7f2546ffd640 1 --2- 172.21.15.92:0/917449321 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2548099660 0x7f2548071a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:06.211 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.208+0000 7f254e174640 1 -- 172.21.15.92:0/917449321 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f2548137870 con 0x7f254812f1c0 2024-09-21T09:43:06.211 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.208+0000 7f254e174640 1 -- 172.21.15.92:0/917449321 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f25481376f0 con 0x7f2548099660 2024-09-21T09:43:06.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.208+0000 7f2547fff640 1 --2- 172.21.15.92:0/917449321 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f254812f1c0 0x7f2548071f40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:06.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.208+0000 7f254e174640 1 -- 172.21.15.92:0/917449321 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f25481379f0 con 0x7f2548098d40 2024-09-21T09:43:06.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.208+0000 7f2546ffd640 1 -- 172.21.15.92:0/917449321 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2548098d40 msgr2=0x7f25480714c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:06.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.208+0000 7f2546ffd640 1 --2- 172.21.15.92:0/917449321 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2548098d40 0x7f25480714c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.208+0000 7f2546ffd640 1 -- 172.21.15.92:0/917449321 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f254812f1c0 msgr2=0x7f2548071f40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:06.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.208+0000 7f2546ffd640 1 --2- 172.21.15.92:0/917449321 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f254812f1c0 0x7f2548071f40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.208+0000 7f2546ffd640 1 -- 172.21.15.92:0/917449321 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2548129460 con 0x7f2548099660 2024-09-21T09:43:06.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.208+0000 7f2546ffd640 1 --2- 172.21.15.92:0/917449321 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2548099660 0x7f2548071a00 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f253c00c3c0 tx=0x7f253c00c880 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:06.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.209+0000 7f252ffff640 1 -- 172.21.15.92:0/917449321 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f253c011790 con 0x7f2548099660 2024-09-21T09:43:06.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.209+0000 7f254e174640 1 -- 172.21.15.92:0/917449321 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f25480742b0 con 0x7f2548099660 2024-09-21T09:43:06.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.209+0000 7f252ffff640 1 -- 172.21.15.92:0/917449321 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f253c017070 con 0x7f2548099660 2024-09-21T09:43:06.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.209+0000 7f254e174640 1 -- 172.21.15.92:0/917449321 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f2548073340 con 0x7f2548099660 2024-09-21T09:43:06.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.209+0000 7f252ffff640 1 -- 172.21.15.92:0/917449321 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f253c011cc0 con 0x7f2548099660 2024-09-21T09:43:06.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.210+0000 7f254e174640 1 -- 172.21.15.92:0/917449321 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2548099120 con 0x7f2548099660 2024-09-21T09:43:06.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.210+0000 7f252ffff640 1 -- 172.21.15.92:0/917449321 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 5) ==== 55999+0+0 (secure 0 0 0) 0x7f253c01c460 con 0x7f2548099660 2024-09-21T09:43:06.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.211+0000 7f25477fe640 1 -- 172.21.15.92:0/917449321 >> v1:172.21.15.92:6800/1280404317 conn(0x7f2534047fd0 legacy=0x7f253404a490 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.92:6800/1280404317 2024-09-21T09:43:06.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.211+0000 7f252ffff640 1 -- 172.21.15.92:0/917449321 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 2812+0+0 (secure 0 0 0) 0x7f253c05d6e0 con 0x7f2548099660 2024-09-21T09:43:06.216 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.214+0000 7f252ffff640 1 -- 172.21.15.92:0/917449321 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+152684 (secure 0 0 0) 0x7f2548129460 con 0x7f2548099660 2024-09-21T09:43:06.222 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.221+0000 7fe684ff2640 1 -- 172.21.15.92:0/2263859485 >> v1:172.21.15.92:6800/1280404317 conn(0x7fe664048020 legacy=0x7fe66404a4e0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.92:6800/1280404317 2024-09-21T09:43:06.259 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.256+0000 7fe675ffb640 1 -- 172.21.15.92:0/2263859485 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mgrmap(e 6) ==== 63244+0+0 (secure 0 0 0) 0x7fe67c07d550 con 0x7fe680098d40 2024-09-21T09:43:06.259 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.257+0000 7fe675ffb640 1 -- 172.21.15.92:0/2263859485 >> v1:172.21.15.92:6800/1280404317 conn(0x7fe664048020 legacy=0x7fe66404a4e0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:43:06.260 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.258+0000 7f252ffff640 1 -- 172.21.15.92:0/917449321 <== mon.1 v2:172.21.15.92:3301/0 7 ==== mgrmap(e 6) ==== 63244+0+0 (secure 0 0 0) 0x7f253c05bfd0 con 0x7f2548099660 2024-09-21T09:43:06.260 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.258+0000 7f252ffff640 1 -- 172.21.15.92:0/917449321 >> v1:172.21.15.92:6800/1280404317 conn(0x7f2534047fd0 legacy=0x7f253404a490 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:43:06.265 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.263+0000 7fd47b7fe640 1 -- 172.21.15.92:0/1331720353 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mgrmap(e 6) ==== 63244+0+0 (secure 0 0 0) 0x7fd48c05bb70 con 0x7fd494074200 2024-09-21T09:43:06.265 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.263+0000 7fd47b7fe640 1 -- 172.21.15.92:0/1331720353 >> v1:172.21.15.92:6800/1280404317 conn(0x7fd480047fd0 legacy=0x7fd48004a490 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:43:06.265 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.263+0000 7fbe9effd640 1 -- 172.21.15.92:0/3925994257 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mgrmap(e 6) ==== 63244+0+0 (secure 0 0 0) 0x7fbea805b560 con 0x7fbeb0071480 2024-09-21T09:43:06.265 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.264+0000 7fbe9effd640 1 -- 172.21.15.92:0/3925994257 >> v1:172.21.15.92:6800/1280404317 conn(0x7fbe90048020 legacy=0x7fbe9004a4e0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:43:06.309 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.307+0000 7fe687a57640 1 -- 172.21.15.92:0/2263859485 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7fe680099120 con 0x7fe680098d40 2024-09-21T09:43:06.309 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.307+0000 7fe675ffb640 1 -- 172.21.15.92:0/2263859485 <== mon.0 v2:172.21.15.92:3300/0 8 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fe67c03f030 con 0x7fe680098d40 2024-09-21T09:43:06.309 INFO:teuthology.orchestra.run.smithi092.stdout:47244640275 2024-09-21T09:43:06.312 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.310+0000 7fe687a57640 1 -- 172.21.15.92:0/2263859485 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe680098d40 msgr2=0x7fe6801c4fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:06.312 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.310+0000 7fe687a57640 1 --2- 172.21.15.92:0/2263859485 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe680098d40 0x7fe6801c4fd0 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7fe67c009780 tx=0x7fe67c033ba0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.312 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.310+0000 7fe687a57640 1 -- 172.21.15.92:0/2263859485 shutdown_connections 2024-09-21T09:43:06.312 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.310+0000 7fe687a57640 1 --2- 172.21.15.92:0/2263859485 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe68012f1c0 0x7fe680071530 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.312 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.310+0000 7fe687a57640 1 --2- 172.21.15.92:0/2263859485 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe680099660 0x7fe6801c5510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.312 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.310+0000 7fe687a57640 1 --2- 172.21.15.92:0/2263859485 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe680098d40 0x7fe6801c4fd0 unknown :-1 s=CLOSED pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.313 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.310+0000 7fe687a57640 1 -- 172.21.15.92:0/2263859485 >> 172.21.15.92:0/2263859485 conn(0x7fe6800a1790 msgr2=0x7fe68009d3d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:06.313 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.311+0000 7fe687a57640 1 -- 172.21.15.92:0/2263859485 shutdown_connections 2024-09-21T09:43:06.313 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.311+0000 7fe687a57640 1 -- 172.21.15.92:0/2263859485 wait complete. 2024-09-21T09:43:06.325 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.323+0000 7fd49903c640 1 -- 172.21.15.92:0/1331720353 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7fd4940008d0 con 0x7fd494074200 2024-09-21T09:43:06.326 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640276 got 47244640275 for osd.0 2024-09-21T09:43:06.326 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.324+0000 7fd47b7fe640 1 -- 172.21.15.92:0/1331720353 <== mon.2 v2:172.21.15.92:3302/0 8 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fd48c00ce30 con 0x7fd494074200 2024-09-21T09:43:06.327 INFO:teuthology.orchestra.run.smithi092.stdout:47244640275 2024-09-21T09:43:06.328 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.326+0000 7fd49903c640 1 -- 172.21.15.92:0/1331720353 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd494074200 msgr2=0x7fd494094ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:06.328 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.326+0000 7fd49903c640 1 --2- 172.21.15.92:0/1331720353 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd494074200 0x7fd494094ec0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7fd48c00c780 tx=0x7fd48c00cc40 comp rx=0 tx=0).stop 2024-09-21T09:43:06.328 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.326+0000 7fd49903c640 1 -- 172.21.15.92:0/1331720353 shutdown_connections 2024-09-21T09:43:06.328 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.326+0000 7fd49903c640 1 --2- 172.21.15.92:0/1331720353 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd494074200 0x7fd494094ec0 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.328 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.326+0000 7fd49903c640 1 --2- 172.21.15.92:0/1331720353 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd494073e20 0x7fd494094980 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.329 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.326+0000 7fd49903c640 1 --2- 172.21.15.92:0/1331720353 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd49406fa10 0x7fd494094440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.329 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.326+0000 7fd49903c640 1 -- 172.21.15.92:0/1331720353 >> 172.21.15.92:0/1331720353 conn(0x7fd49409dde0 msgr2=0x7fd4941362f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:06.329 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.327+0000 7fd49903c640 1 -- 172.21.15.92:0/1331720353 shutdown_connections 2024-09-21T09:43:06.329 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.327+0000 7fd49903c640 1 -- 172.21.15.92:0/1331720353 wait complete. 2024-09-21T09:43:06.339 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640276 got 47244640275 for osd.2 2024-09-21T09:43:06.373 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.372+0000 7fbe9cff9640 1 -- 172.21.15.92:0/3925994257 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7fbeb00698c0 con 0x7fbeb0071480 2024-09-21T09:43:06.374 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.372+0000 7fbe9effd640 1 -- 172.21.15.92:0/3925994257 <== mon.2 v2:172.21.15.92:3302/0 8 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fbea8016170 con 0x7fbeb0071480 2024-09-21T09:43:06.374 INFO:teuthology.orchestra.run.smithi092.stdout:47244640273 2024-09-21T09:43:06.376 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.374+0000 7fbe9cff9640 1 -- 172.21.15.92:0/3925994257 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbeb0071480 msgr2=0x7fbeb0070190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:06.376 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.374+0000 7fbe9cff9640 1 --2- 172.21.15.92:0/3925994257 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbeb0071480 0x7fbeb0070190 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7fbea800c670 tx=0x7fbea800cb30 comp rx=0 tx=0).stop 2024-09-21T09:43:06.376 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.374+0000 7fbe9cff9640 1 -- 172.21.15.92:0/3925994257 shutdown_connections 2024-09-21T09:43:06.376 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.374+0000 7fbe9cff9640 1 --2- 172.21.15.92:0/3925994257 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbeb0071480 0x7fbeb0070190 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.376 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.374+0000 7fbe9cff9640 1 --2- 172.21.15.92:0/3925994257 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbeb0091510 0x7fbeb006fc50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.377 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.374+0000 7fbe9cff9640 1 --2- 172.21.15.92:0/3925994257 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbeb006ebf0 0x7fbeb006f710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.377 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.374+0000 7fbe9cff9640 1 -- 172.21.15.92:0/3925994257 >> 172.21.15.92:0/3925994257 conn(0x7fbeb009e490 msgr2=0x7fbeb012c640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:06.377 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.374+0000 7fbe9cff9640 1 -- 172.21.15.92:0/3925994257 shutdown_connections 2024-09-21T09:43:06.377 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.374+0000 7fbe9cff9640 1 -- 172.21.15.92:0/3925994257 wait complete. 2024-09-21T09:43:06.387 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640274 got 47244640273 for osd.1 2024-09-21T09:43:06.394 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.392+0000 7f254e174640 1 -- 172.21.15.92:0/917449321 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7f254812ad80 con 0x7f2548099660 2024-09-21T09:43:06.394 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.393+0000 7f252ffff640 1 -- 172.21.15.92:0/917449321 <== mon.1 v2:172.21.15.92:3301/0 8 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f253c03a5a0 con 0x7f2548099660 2024-09-21T09:43:06.394 INFO:teuthology.orchestra.run.smithi092.stdout:38654705682 2024-09-21T09:43:06.397 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.395+0000 7f254e174640 1 -- 172.21.15.92:0/917449321 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2548099660 msgr2=0x7f2548071a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:06.397 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.395+0000 7f254e174640 1 --2- 172.21.15.92:0/917449321 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2548099660 0x7f2548071a00 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f253c00c3c0 tx=0x7f253c00c880 comp rx=0 tx=0).stop 2024-09-21T09:43:06.397 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.395+0000 7f254e174640 1 -- 172.21.15.92:0/917449321 shutdown_connections 2024-09-21T09:43:06.397 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.395+0000 7f254e174640 1 --2- 172.21.15.92:0/917449321 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f254812f1c0 0x7f2548071f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.397 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.395+0000 7f254e174640 1 --2- 172.21.15.92:0/917449321 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2548099660 0x7f2548071a00 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.397 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.395+0000 7f254e174640 1 --2- 172.21.15.92:0/917449321 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2548098d40 0x7f25480714c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:06.397 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.395+0000 7f254e174640 1 -- 172.21.15.92:0/917449321 >> 172.21.15.92:0/917449321 conn(0x7f25480a1790 msgr2=0x7f254809d1f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:06.397 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.395+0000 7f254e174640 1 -- 172.21.15.92:0/917449321 shutdown_connections 2024-09-21T09:43:06.397 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:06.395+0000 7f254e174640 1 -- 172.21.15.92:0/917449321 wait complete. 2024-09-21T09:43:06.407 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705682 for osd.3 2024-09-21T09:43:07.259 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:43:07.258+0000 7f4e27b9b640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-21T09:43:07.261 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:43:07.260+0000 7f4e27b9b640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-21T09:43:07.262 INFO:tasks.ceph.mgr.y.smithi092.stderr:2024-09-21T09:43:07.260+0000 7f4e27b9b640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-21T09:43:07.327 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T09:43:07.340 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T09:43:07.387 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T09:43:07.407 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T09:43:07.442 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.439+0000 7fcbaaa9a640 1 -- 172.21.15.92:0/870477252 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcba4129f60 msgr2=0x7fcba412c880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:07.442 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.439+0000 7fcbaaa9a640 1 --2- 172.21.15.92:0/870477252 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcba4129f60 0x7fcba412c880 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7fcb90005fd0 tx=0x7fcb9002a4b0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.442 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.440+0000 7fcbaaa9a640 1 -- 172.21.15.92:0/870477252 shutdown_connections 2024-09-21T09:43:07.442 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.440+0000 7fcbaaa9a640 1 --2- 172.21.15.92:0/870477252 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcba412cdc0 0x7fcba412d250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.442 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.440+0000 7fcbaaa9a640 1 --2- 172.21.15.92:0/870477252 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcba4129f60 0x7fcba412c880 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.442 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.440+0000 7fcbaaa9a640 1 --2- 172.21.15.92:0/870477252 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcba4129640 0x7fcba4129a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.442 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.440+0000 7fcbaaa9a640 1 -- 172.21.15.92:0/870477252 >> 172.21.15.92:0/870477252 conn(0x7fcba409e610 msgr2=0x7fcba409ea20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:07.442 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.440+0000 7fcbaaa9a640 1 -- 172.21.15.92:0/870477252 shutdown_connections 2024-09-21T09:43:07.442 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.440+0000 7fcbaaa9a640 1 -- 172.21.15.92:0/870477252 wait complete. 2024-09-21T09:43:07.443 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.441+0000 7fcbaaa9a640 1 Processor -- start 2024-09-21T09:43:07.443 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.441+0000 7fcbaaa9a640 1 -- start start 2024-09-21T09:43:07.450 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.442+0000 7fcbaaa9a640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcba4129640 0x7fcba4131680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:07.450 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.442+0000 7fcbaaa9a640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcba4129f60 0x7fcba4131bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:07.450 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.442+0000 7fcbaaa9a640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcba412cdc0 0x7fcba4132100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:07.450 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.442+0000 7fcbaaa9a640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fcba41343a0 con 0x7fcba4129f60 2024-09-21T09:43:07.450 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.442+0000 7fcbaaa9a640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fcba4134220 con 0x7fcba412cdc0 2024-09-21T09:43:07.450 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.442+0000 7fcbaaa9a640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fcba4134520 con 0x7fcba4129640 2024-09-21T09:43:07.450 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.442+0000 7fcba8836640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcba4129640 0x7fcba4131680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:07.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.442+0000 7fcba8836640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcba4129640 0x7fcba4131680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:49170/0 (socket says 172.21.15.92:49170) 2024-09-21T09:43:07.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.442+0000 7fcba8836640 1 -- 172.21.15.92:0/1943158988 learned_addr learned my addr 172.21.15.92:0/1943158988 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:07.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.442+0000 7fcba9037640 1 --2- 172.21.15.92:0/1943158988 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcba412cdc0 0x7fcba4132100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:07.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.442+0000 7fcba3fff640 1 --2- 172.21.15.92:0/1943158988 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcba4129f60 0x7fcba4131bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:07.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.442+0000 7fcba8836640 1 -- 172.21.15.92:0/1943158988 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcba412cdc0 msgr2=0x7fcba4132100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:07.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.442+0000 7fcba8836640 1 --2- 172.21.15.92:0/1943158988 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcba412cdc0 0x7fcba4132100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.442+0000 7fcba8836640 1 -- 172.21.15.92:0/1943158988 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcba4129f60 msgr2=0x7fcba4131bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:07.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.442+0000 7fcba8836640 1 --2- 172.21.15.92:0/1943158988 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcba4129f60 0x7fcba4131bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.442+0000 7fcba8836640 1 -- 172.21.15.92:0/1943158988 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcba41232b0 con 0x7fcba4129640 2024-09-21T09:43:07.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.442+0000 7fcba3fff640 1 --2- 172.21.15.92:0/1943158988 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcba4129f60 0x7fcba4131bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:43:07.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.443+0000 7fcba8836640 1 --2- 172.21.15.92:0/1943158988 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcba4129640 0x7fcba4131680 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7fcb9800ea60 tx=0x7fcb9800ef20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:07.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.443+0000 7fcba17fa640 1 -- 172.21.15.92:0/1943158988 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcb9800cde0 con 0x7fcba4129640 2024-09-21T09:43:07.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.443+0000 7fcbaaa9a640 1 -- 172.21.15.92:0/1943158988 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fcba41242d0 con 0x7fcba4129640 2024-09-21T09:43:07.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.443+0000 7fcba17fa640 1 -- 172.21.15.92:0/1943158988 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fcb98004020 con 0x7fcba4129640 2024-09-21T09:43:07.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.443+0000 7fcbaaa9a640 1 -- 172.21.15.92:0/1943158988 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fcba4133590 con 0x7fcba4129640 2024-09-21T09:43:07.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.443+0000 7fcba17fa640 1 -- 172.21.15.92:0/1943158988 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcb980043f0 con 0x7fcba4129640 2024-09-21T09:43:07.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.445+0000 7fcba17fa640 1 -- 172.21.15.92:0/1943158988 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 7) ==== 63336+0+0 (secure 0 0 0) 0x7fcb980035f0 con 0x7fcba4129640 2024-09-21T09:43:07.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.445+0000 7fcba17fa640 1 -- 172.21.15.92:0/1943158988 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7fcb98069680 con 0x7fcba4129640 2024-09-21T09:43:07.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.445+0000 7fcbaaa9a640 1 -- 172.21.15.92:0/1943158988 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fcba4073cd0 con 0x7fcba4129640 2024-09-21T09:43:07.453 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.449+0000 7fcba17fa640 1 -- 172.21.15.92:0/1943158988 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fcb980684b0 con 0x7fcba4129640 2024-09-21T09:43:07.481 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.479+0000 7fd017df9640 1 -- 172.21.15.92:0/1464981678 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcffc004800 msgr2=0x7fcffc004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:07.481 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.479+0000 7fd017df9640 1 --2- 172.21.15.92:0/1464981678 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcffc004800 0x7fcffc004c80 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7fd008009f90 tx=0x7fcffc005830 comp rx=0 tx=0).stop 2024-09-21T09:43:07.481 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.479+0000 7fd017df9640 1 -- 172.21.15.92:0/1464981678 shutdown_connections 2024-09-21T09:43:07.481 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.479+0000 7fd017df9640 1 --2- 172.21.15.92:0/1464981678 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcffc004800 0x7fcffc004c80 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.481 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.479+0000 7fd017df9640 1 --2- 172.21.15.92:0/1464981678 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd010093310 0x7fcffc004010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.481 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.479+0000 7fd017df9640 1 --2- 172.21.15.92:0/1464981678 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd010131080 0x7fcffc0039e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.481 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.479+0000 7fd017df9640 1 -- 172.21.15.92:0/1464981678 >> 172.21.15.92:0/1464981678 conn(0x7fd01009e490 msgr2=0x7fd01009e8a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:07.481 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.479+0000 7fd017df9640 1 -- 172.21.15.92:0/1464981678 shutdown_connections 2024-09-21T09:43:07.481 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.479+0000 7fd017df9640 1 -- 172.21.15.92:0/1464981678 wait complete. 2024-09-21T09:43:07.482 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.480+0000 7fd017df9640 1 Processor -- start 2024-09-21T09:43:07.482 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.480+0000 7fd017df9640 1 -- start start 2024-09-21T09:43:07.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.481+0000 7fd017df9640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcffc004800 0x7fd0101be2f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:07.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.481+0000 7fd017df9640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd010092820 0x7fd0101be830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:07.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.481+0000 7fd015b95640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcffc004800 0x7fd0101be2f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:07.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.481+0000 7fd015b95640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcffc004800 0x7fd0101be2f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:53474/0 (socket says 172.21.15.92:53474) 2024-09-21T09:43:07.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.481+0000 7fd015394640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd010092820 0x7fd0101be830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:07.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.482+0000 7fd015394640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd010092820 0x7fd0101be830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:49186/0 (socket says 172.21.15.92:49186) 2024-09-21T09:43:07.485 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.482+0000 7fd017df9640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd010093310 0x7fd0101bed70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:07.485 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.482+0000 7fd016396640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd010093310 0x7fd0101bed70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:07.485 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.482+0000 7fd016396640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd010093310 0x7fd0101bed70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:59494/0 (socket says 172.21.15.92:59494) 2024-09-21T09:43:07.485 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.482+0000 7fd015b95640 1 -- 172.21.15.92:0/507957008 learned_addr learned my addr 172.21.15.92:0/507957008 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:07.485 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.482+0000 7fd017df9640 1 -- 172.21.15.92:0/507957008 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fd01006ee40 con 0x7fcffc004800 2024-09-21T09:43:07.485 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.482+0000 7fd017df9640 1 -- 172.21.15.92:0/507957008 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fd01006ecc0 con 0x7fd010093310 2024-09-21T09:43:07.485 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.482+0000 7fd017df9640 1 -- 172.21.15.92:0/507957008 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fd01006efc0 con 0x7fd010092820 2024-09-21T09:43:07.485 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.482+0000 7fd015b95640 1 -- 172.21.15.92:0/507957008 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd010092820 msgr2=0x7fd0101be830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:07.487 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.482+0000 7fd015b95640 1 --2- 172.21.15.92:0/507957008 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd010092820 0x7fd0101be830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.487 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.482+0000 7fd015b95640 1 -- 172.21.15.92:0/507957008 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd010093310 msgr2=0x7fd0101bed70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:07.487 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.482+0000 7fd015b95640 1 --2- 172.21.15.92:0/507957008 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd010093310 0x7fd0101bed70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.487 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.482+0000 7fd015b95640 1 -- 172.21.15.92:0/507957008 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd010129050 con 0x7fcffc004800 2024-09-21T09:43:07.487 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.483+0000 7fd016396640 1 --2- 172.21.15.92:0/507957008 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd010093310 0x7fd0101bed70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:43:07.487 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.483+0000 7fd015b95640 1 --2- 172.21.15.92:0/507957008 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcffc004800 0x7fd0101be2f0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7fd00c001160 tx=0x7fd00c0039c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:07.487 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.483+0000 7fd0067fc640 1 -- 172.21.15.92:0/507957008 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd00c015070 con 0x7fcffc004800 2024-09-21T09:43:07.487 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.483+0000 7fd017df9640 1 -- 172.21.15.92:0/507957008 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd010096a00 con 0x7fcffc004800 2024-09-21T09:43:07.487 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.483+0000 7fd017df9640 1 -- 172.21.15.92:0/507957008 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd0101bf370 con 0x7fcffc004800 2024-09-21T09:43:07.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.483+0000 7fd0067fc640 1 -- 172.21.15.92:0/507957008 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fd00c011900 con 0x7fcffc004800 2024-09-21T09:43:07.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.483+0000 7fd0067fc640 1 -- 172.21.15.92:0/507957008 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd00c01f060 con 0x7fcffc004800 2024-09-21T09:43:07.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.484+0000 7fd017df9640 1 -- 172.21.15.92:0/507957008 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fcff8005290 con 0x7fcffc004800 2024-09-21T09:43:07.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.485+0000 7fd0067fc640 1 -- 172.21.15.92:0/507957008 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 7) ==== 63336+0+0 (secure 0 0 0) 0x7fd00c018070 con 0x7fcffc004800 2024-09-21T09:43:07.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.485+0000 7fd0067fc640 1 -- 172.21.15.92:0/507957008 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7fd00c069b90 con 0x7fcffc004800 2024-09-21T09:43:07.494 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.492+0000 7fd0067fc640 1 -- 172.21.15.92:0/507957008 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fd00c0689c0 con 0x7fcffc004800 2024-09-21T09:43:07.524 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.522+0000 7f17c069a640 1 -- 172.21.15.92:0/805077836 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f17b812f1c0 msgr2=0x7f17b812f600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:07.525 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.522+0000 7f17c069a640 1 --2- 172.21.15.92:0/805077836 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f17b812f1c0 0x7f17b812f600 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f17b0005fd0 tx=0x7f17b002a780 comp rx=0 tx=0).stop 2024-09-21T09:43:07.525 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.522+0000 7f17c069a640 1 -- 172.21.15.92:0/805077836 shutdown_connections 2024-09-21T09:43:07.525 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.522+0000 7f17c069a640 1 --2- 172.21.15.92:0/805077836 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f17b812f1c0 0x7f17b812f600 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.525 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.522+0000 7f17c069a640 1 --2- 172.21.15.92:0/805077836 >> v2:172.21.15.92:3300/0 conn(0x7f17b8098d40 0x7f17b8099120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.525 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.522+0000 7f17c069a640 1 -- 172.21.15.92:0/805077836 >> 172.21.15.92:0/805077836 conn(0x7f17b80a1790 msgr2=0x7f17b80a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:07.525 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.523+0000 7f17c069a640 1 -- 172.21.15.92:0/805077836 shutdown_connections 2024-09-21T09:43:07.525 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.523+0000 7f17c069a640 1 -- 172.21.15.92:0/805077836 wait complete. 2024-09-21T09:43:07.528 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.524+0000 7f17c069a640 1 Processor -- start 2024-09-21T09:43:07.528 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.524+0000 7f17c069a640 1 -- start start 2024-09-21T09:43:07.528 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.524+0000 7f17c069a640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f17b8098d40 0x7f17b81c4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:07.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.524+0000 7f17c069a640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f17b8099660 0x7f17b81c5470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:07.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.524+0000 7f17c069a640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f17b812f1c0 0x7f17b8071370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:07.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.524+0000 7f17c069a640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f17b8137870 con 0x7f17b812f1c0 2024-09-21T09:43:07.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.524+0000 7f17c069a640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f17b81376f0 con 0x7f17b8099660 2024-09-21T09:43:07.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.524+0000 7f17c069a640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f17b81379f0 con 0x7f17b8098d40 2024-09-21T09:43:07.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.524+0000 7f17be436640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f17b8098d40 0x7f17b81c4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:07.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.524+0000 7f17be436640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f17b8098d40 0x7f17b81c4f30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:49216/0 (socket says 172.21.15.92:49216) 2024-09-21T09:43:07.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.524+0000 7f17be436640 1 -- 172.21.15.92:0/1279854899 learned_addr learned my addr 172.21.15.92:0/1279854899 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:07.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.524+0000 7f17bdc35640 1 --2- 172.21.15.92:0/1279854899 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f17b8099660 0x7f17b81c5470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:07.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.524+0000 7f17bec37640 1 --2- 172.21.15.92:0/1279854899 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f17b812f1c0 0x7f17b8071370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:07.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.525+0000 7f17be436640 1 -- 172.21.15.92:0/1279854899 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f17b8099660 msgr2=0x7f17b81c5470 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:07.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.525+0000 7f17be436640 1 --2- 172.21.15.92:0/1279854899 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f17b8099660 0x7f17b81c5470 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.525+0000 7f17be436640 1 -- 172.21.15.92:0/1279854899 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f17b812f1c0 msgr2=0x7f17b8071370 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:07.530 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.525+0000 7f17be436640 1 --2- 172.21.15.92:0/1279854899 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f17b812f1c0 0x7f17b8071370 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.530 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.525+0000 7f17be436640 1 -- 172.21.15.92:0/1279854899 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f17b80733e0 con 0x7f17b8098d40 2024-09-21T09:43:07.530 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.525+0000 7f17bec37640 1 --2- 172.21.15.92:0/1279854899 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f17b812f1c0 0x7f17b8071370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:07.530 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.525+0000 7f17be436640 1 --2- 172.21.15.92:0/1279854899 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f17b8098d40 0x7f17b81c4f30 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f17b400e790 tx=0x7f17b400ec50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:07.530 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.525+0000 7f17aeffd640 1 -- 172.21.15.92:0/1279854899 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f17b400eea0 con 0x7f17b8098d40 2024-09-21T09:43:07.530 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.525+0000 7f17c069a640 1 -- 172.21.15.92:0/1279854899 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f17b812fe00 con 0x7f17b8098d40 2024-09-21T09:43:07.530 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.525+0000 7f17aeffd640 1 -- 172.21.15.92:0/1279854899 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f17b400cdb0 con 0x7f17b8098d40 2024-09-21T09:43:07.530 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.526+0000 7f17c069a640 1 -- 172.21.15.92:0/1279854899 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f17b81c59b0 con 0x7f17b8098d40 2024-09-21T09:43:07.530 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.526+0000 7f17aeffd640 1 -- 172.21.15.92:0/1279854899 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f17b4010690 con 0x7f17b8098d40 2024-09-21T09:43:07.530 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.526+0000 7f17c069a640 1 -- 172.21.15.92:0/1279854899 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f17b812ead0 con 0x7f17b8098d40 2024-09-21T09:43:07.531 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.528+0000 7f17aeffd640 1 -- 172.21.15.92:0/1279854899 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 7) ==== 63336+0+0 (secure 0 0 0) 0x7f17b4010830 con 0x7f17b8098d40 2024-09-21T09:43:07.531 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.529+0000 7f17aeffd640 1 -- 172.21.15.92:0/1279854899 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7f17b4068e50 con 0x7f17b8098d40 2024-09-21T09:43:07.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.531+0000 7f17aeffd640 1 -- 172.21.15.92:0/1279854899 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f17b812ead0 con 0x7f17b8098d40 2024-09-21T09:43:07.565 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.563+0000 7f2e18e46640 1 -- 172.21.15.92:0/1278189092 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2e14131380 msgr2=0x7f2e14133770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:07.565 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.563+0000 7f2e18e46640 1 --2- 172.21.15.92:0/1278189092 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2e14131380 0x7f2e14133770 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f2e0c005fd0 tx=0x7f2e0c02a780 comp rx=0 tx=0).stop 2024-09-21T09:43:07.566 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.563+0000 7f2e18e46640 1 -- 172.21.15.92:0/1278189092 shutdown_connections 2024-09-21T09:43:07.566 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.563+0000 7f2e18e46640 1 --2- 172.21.15.92:0/1278189092 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2e14131380 0x7f2e14133770 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.566 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.563+0000 7f2e18e46640 1 --2- 172.21.15.92:0/1278189092 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2e1412ea50 0x7f2e14130e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.566 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.564+0000 7f2e18e46640 1 -- 172.21.15.92:0/1278189092 >> 172.21.15.92:0/1278189092 conn(0x7f2e1409c470 msgr2=0x7f2e1409c880 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:07.566 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.564+0000 7f2e18e46640 1 -- 172.21.15.92:0/1278189092 shutdown_connections 2024-09-21T09:43:07.566 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.564+0000 7f2e18e46640 1 -- 172.21.15.92:0/1278189092 wait complete. 2024-09-21T09:43:07.566 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.565+0000 7f2e18e46640 1 Processor -- start 2024-09-21T09:43:07.567 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.565+0000 7f2e18e46640 1 -- start start 2024-09-21T09:43:07.569 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.565+0000 7f2e18e46640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2e14095db0 0x7f2e14134cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:07.569 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.565+0000 7f2e18e46640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2e1412ea50 0x7f2e14135200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:07.569 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.565+0000 7f2e18e46640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2e14131380 0x7f2e14135740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:07.569 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.565+0000 7f2e18e46640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f2e141345c0 con 0x7f2e14131380 2024-09-21T09:43:07.569 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.565+0000 7f2e18e46640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f2e14134440 con 0x7f2e14095db0 2024-09-21T09:43:07.569 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.565+0000 7f2e18e46640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f2e14134740 con 0x7f2e1412ea50 2024-09-21T09:43:07.569 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.565+0000 7f2e1259c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2e14095db0 0x7f2e14134cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:07.569 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.565+0000 7f2e1259c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2e14095db0 0x7f2e14134cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:59524/0 (socket says 172.21.15.92:59524) 2024-09-21T09:43:07.569 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.566+0000 7f2e1259c640 1 -- 172.21.15.92:0/2905023832 learned_addr learned my addr 172.21.15.92:0/2905023832 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:07.569 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.566+0000 7f2e11d9b640 1 --2- 172.21.15.92:0/2905023832 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2e1412ea50 0x7f2e14135200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:07.570 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.566+0000 7f2e12d9d640 1 --2- 172.21.15.92:0/2905023832 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2e14131380 0x7f2e14135740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:07.570 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.566+0000 7f2e12d9d640 1 -- 172.21.15.92:0/2905023832 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2e1412ea50 msgr2=0x7f2e14135200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:07.570 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.566+0000 7f2e12d9d640 1 --2- 172.21.15.92:0/2905023832 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2e1412ea50 0x7f2e14135200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.570 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.566+0000 7f2e12d9d640 1 -- 172.21.15.92:0/2905023832 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2e14095db0 msgr2=0x7f2e14134cc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:07.570 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.566+0000 7f2e12d9d640 1 --2- 172.21.15.92:0/2905023832 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2e14095db0 0x7f2e14134cc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.570 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.566+0000 7f2e12d9d640 1 -- 172.21.15.92:0/2905023832 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2e141360a0 con 0x7f2e14131380 2024-09-21T09:43:07.570 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.566+0000 7f2e12d9d640 1 --2- 172.21.15.92:0/2905023832 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2e14131380 0x7f2e14135740 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f2e0c009000 tx=0x7f2e0c003c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:07.570 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.567+0000 7f2e02ffd640 1 -- 172.21.15.92:0/2905023832 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e0c040070 con 0x7f2e14131380 2024-09-21T09:43:07.570 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.567+0000 7f2e02ffd640 1 -- 172.21.15.92:0/2905023832 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2e0c004540 con 0x7f2e14131380 2024-09-21T09:43:07.570 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.567+0000 7f2e02ffd640 1 -- 172.21.15.92:0/2905023832 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e0c03c420 con 0x7f2e14131380 2024-09-21T09:43:07.570 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.567+0000 7f2e18e46640 1 -- 172.21.15.92:0/2905023832 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2e14124160 con 0x7f2e14131380 2024-09-21T09:43:07.571 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.567+0000 7f2e18e46640 1 -- 172.21.15.92:0/2905023832 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2e14136aa0 con 0x7f2e14131380 2024-09-21T09:43:07.571 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.569+0000 7f2e02ffd640 1 -- 172.21.15.92:0/2905023832 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 7) ==== 63336+0+0 (secure 0 0 0) 0x7f2e0c043070 con 0x7f2e14131380 2024-09-21T09:43:07.571 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.569+0000 7f2e18e46640 1 -- 172.21.15.92:0/2905023832 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2e14096190 con 0x7f2e14131380 2024-09-21T09:43:07.571 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.569+0000 7f2e02ffd640 1 -- 172.21.15.92:0/2905023832 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7f2e0c08a1e0 con 0x7f2e14131380 2024-09-21T09:43:07.578 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.577+0000 7f2e02ffd640 1 -- 172.21.15.92:0/2905023832 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f2e14124160 con 0x7f2e14131380 2024-09-21T09:43:07.635 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.634+0000 7fcbaaa9a640 1 -- 172.21.15.92:0/1943158988 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7fcba4073740 con 0x7fcba4129640 2024-09-21T09:43:07.636 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.634+0000 7fcba17fa640 1 -- 172.21.15.92:0/1943158988 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fcb98042400 con 0x7fcba4129640 2024-09-21T09:43:07.636 INFO:teuthology.orchestra.run.smithi092.stdout:47244640275 2024-09-21T09:43:07.639 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.637+0000 7fcbaaa9a640 1 -- 172.21.15.92:0/1943158988 >> v1:172.21.15.92:6800/2299163855 conn(0x7fcb88051300 legacy=0x7fcb880537c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:07.639 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.637+0000 7fcbaaa9a640 1 -- 172.21.15.92:0/1943158988 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcba4129640 msgr2=0x7fcba4131680 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:07.639 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.637+0000 7fcbaaa9a640 1 --2- 172.21.15.92:0/1943158988 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcba4129640 0x7fcba4131680 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7fcb9800ea60 tx=0x7fcb9800ef20 comp rx=0 tx=0).stop 2024-09-21T09:43:07.639 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.637+0000 7fcbaaa9a640 1 -- 172.21.15.92:0/1943158988 shutdown_connections 2024-09-21T09:43:07.639 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.637+0000 7fcbaaa9a640 1 --2- 172.21.15.92:0/1943158988 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcba412cdc0 0x7fcba4132100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.639 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.637+0000 7fcbaaa9a640 1 --2- 172.21.15.92:0/1943158988 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcba4129f60 0x7fcba4131bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.639 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.637+0000 7fcbaaa9a640 1 --2- 172.21.15.92:0/1943158988 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcba4129640 0x7fcba4131680 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.639 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.637+0000 7fcbaaa9a640 1 -- 172.21.15.92:0/1943158988 >> 172.21.15.92:0/1943158988 conn(0x7fcba409e610 msgr2=0x7fcba412b840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:07.639 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.637+0000 7fcbaaa9a640 1 -- 172.21.15.92:0/1943158988 shutdown_connections 2024-09-21T09:43:07.640 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.637+0000 7fcbaaa9a640 1 -- 172.21.15.92:0/1943158988 wait complete. 2024-09-21T09:43:07.654 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640276 got 47244640275 for osd.0 2024-09-21T09:43:07.663 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.662+0000 7fd017df9640 1 -- 172.21.15.92:0/507957008 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7fcff8005580 con 0x7fcffc004800 2024-09-21T09:43:07.664 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.662+0000 7fd0067fc640 1 -- 172.21.15.92:0/507957008 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fd00c042910 con 0x7fcffc004800 2024-09-21T09:43:07.664 INFO:teuthology.orchestra.run.smithi092.stdout:47244640275 2024-09-21T09:43:07.666 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.664+0000 7fd017df9640 1 -- 172.21.15.92:0/507957008 >> v1:172.21.15.92:6800/2299163855 conn(0x7fcff4055710 legacy=0x7fcff4057bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:07.666 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.664+0000 7fd017df9640 1 -- 172.21.15.92:0/507957008 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcffc004800 msgr2=0x7fd0101be2f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:07.666 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.664+0000 7fd017df9640 1 --2- 172.21.15.92:0/507957008 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcffc004800 0x7fd0101be2f0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7fd00c001160 tx=0x7fd00c0039c0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.666 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.664+0000 7fd017df9640 1 -- 172.21.15.92:0/507957008 shutdown_connections 2024-09-21T09:43:07.666 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.664+0000 7fd017df9640 1 --2- 172.21.15.92:0/507957008 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd010093310 0x7fd0101bed70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.666 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.664+0000 7fd017df9640 1 --2- 172.21.15.92:0/507957008 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd010092820 0x7fd0101be830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.666 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.664+0000 7fd017df9640 1 --2- 172.21.15.92:0/507957008 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcffc004800 0x7fd0101be2f0 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.666 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.664+0000 7fd017df9640 1 -- 172.21.15.92:0/507957008 >> 172.21.15.92:0/507957008 conn(0x7fd01009e490 msgr2=0x7fd010123130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:07.667 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.664+0000 7fd017df9640 1 -- 172.21.15.92:0/507957008 shutdown_connections 2024-09-21T09:43:07.667 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.664+0000 7fd017df9640 1 -- 172.21.15.92:0/507957008 wait complete. 2024-09-21T09:43:07.676 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640276 got 47244640275 for osd.2 2024-09-21T09:43:07.702 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.701+0000 7f17c069a640 1 -- 172.21.15.92:0/1279854899 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7f17b8099120 con 0x7f17b8098d40 2024-09-21T09:43:07.703 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.701+0000 7f17aeffd640 1 -- 172.21.15.92:0/1279854899 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f17b812ead0 con 0x7f17b8098d40 2024-09-21T09:43:07.703 INFO:teuthology.orchestra.run.smithi092.stdout:47244640273 2024-09-21T09:43:07.705 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.703+0000 7f17c069a640 1 -- 172.21.15.92:0/1279854899 >> v1:172.21.15.92:6800/2299163855 conn(0x7f179c051330 legacy=0x7f179c0537f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:07.705 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.703+0000 7f17c069a640 1 -- 172.21.15.92:0/1279854899 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f17b8098d40 msgr2=0x7f17b81c4f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:07.705 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.703+0000 7f17c069a640 1 --2- 172.21.15.92:0/1279854899 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f17b8098d40 0x7f17b81c4f30 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f17b400e790 tx=0x7f17b400ec50 comp rx=0 tx=0).stop 2024-09-21T09:43:07.705 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.703+0000 7f17c069a640 1 -- 172.21.15.92:0/1279854899 shutdown_connections 2024-09-21T09:43:07.705 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.703+0000 7f17c069a640 1 --2- 172.21.15.92:0/1279854899 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f17b812f1c0 0x7f17b8071370 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.705 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.703+0000 7f17c069a640 1 --2- 172.21.15.92:0/1279854899 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f17b8099660 0x7f17b81c5470 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.706 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.703+0000 7f17c069a640 1 --2- 172.21.15.92:0/1279854899 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f17b8098d40 0x7f17b81c4f30 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.706 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.703+0000 7f17c069a640 1 -- 172.21.15.92:0/1279854899 >> 172.21.15.92:0/1279854899 conn(0x7f17b80a1790 msgr2=0x7f17b809cb70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:07.706 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.703+0000 7f17c069a640 1 -- 172.21.15.92:0/1279854899 shutdown_connections 2024-09-21T09:43:07.706 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.703+0000 7f17c069a640 1 -- 172.21.15.92:0/1279854899 wait complete. 2024-09-21T09:43:07.715 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640274 got 47244640273 for osd.1 2024-09-21T09:43:07.737 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.735+0000 7f2e18e46640 1 -- 172.21.15.92:0/2905023832 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7f2e14132e70 con 0x7f2e14131380 2024-09-21T09:43:07.737 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.735+0000 7f2e02ffd640 1 -- 172.21.15.92:0/2905023832 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f2e0c049d90 con 0x7f2e14131380 2024-09-21T09:43:07.737 INFO:teuthology.orchestra.run.smithi092.stdout:38654705682 2024-09-21T09:43:07.739 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.737+0000 7f2e18e46640 1 -- 172.21.15.92:0/2905023832 >> v1:172.21.15.92:6800/2299163855 conn(0x7f2df4051300 legacy=0x7f2df40537c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:07.739 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.737+0000 7f2e18e46640 1 -- 172.21.15.92:0/2905023832 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2e14131380 msgr2=0x7f2e14135740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:07.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.737+0000 7f2e18e46640 1 --2- 172.21.15.92:0/2905023832 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2e14131380 0x7f2e14135740 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f2e0c009000 tx=0x7f2e0c003c10 comp rx=0 tx=0).stop 2024-09-21T09:43:07.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.737+0000 7f2e18e46640 1 -- 172.21.15.92:0/2905023832 shutdown_connections 2024-09-21T09:43:07.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.737+0000 7f2e18e46640 1 --2- 172.21.15.92:0/2905023832 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2e14131380 0x7f2e14135740 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.737+0000 7f2e18e46640 1 --2- 172.21.15.92:0/2905023832 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2e1412ea50 0x7f2e14135200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.737+0000 7f2e18e46640 1 --2- 172.21.15.92:0/2905023832 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2e14095db0 0x7f2e14134cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:07.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.738+0000 7f2e18e46640 1 -- 172.21.15.92:0/2905023832 >> 172.21.15.92:0/2905023832 conn(0x7f2e1409c470 msgr2=0x7f2e1412dcd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:07.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.738+0000 7f2e18e46640 1 -- 172.21.15.92:0/2905023832 shutdown_connections 2024-09-21T09:43:07.740 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:07.738+0000 7f2e18e46640 1 -- 172.21.15.92:0/2905023832 wait complete. 2024-09-21T09:43:07.750 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705682 for osd.3 2024-09-21T09:43:08.655 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T09:43:08.677 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T09:43:08.716 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T09:43:08.750 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T09:43:08.760 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.758+0000 7fd0a5ccb640 1 -- 172.21.15.92:0/1654254020 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd0a00715f0 msgr2=0x7fd0a00719d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:08.760 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.758+0000 7fd0a5ccb640 1 --2- 172.21.15.92:0/1654254020 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd0a00715f0 0x7fd0a00719d0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fd090005fd0 tx=0x7fd09002a410 comp rx=0 tx=0).stop 2024-09-21T09:43:08.760 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.758+0000 7fd0a5ccb640 1 -- 172.21.15.92:0/1654254020 shutdown_connections 2024-09-21T09:43:08.760 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.758+0000 7fd0a5ccb640 1 --2- 172.21.15.92:0/1654254020 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd0a0123090 0x7fd0a0123500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.760 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.758+0000 7fd0a5ccb640 1 --2- 172.21.15.92:0/1654254020 >> v2:172.21.15.92:3300/0 conn(0x7fd0a0071f10 0x7fd0a0122b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.758+0000 7fd0a5ccb640 1 --2- 172.21.15.92:0/1654254020 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd0a00715f0 0x7fd0a00719d0 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.758+0000 7fd0a5ccb640 1 -- 172.21.15.92:0/1654254020 >> 172.21.15.92:0/1654254020 conn(0x7fd0a009d3a0 msgr2=0x7fd0a009d7b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:08.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.758+0000 7fd0a5ccb640 1 -- 172.21.15.92:0/1654254020 shutdown_connections 2024-09-21T09:43:08.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.758+0000 7fd0a5ccb640 1 -- 172.21.15.92:0/1654254020 wait complete. 2024-09-21T09:43:08.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.759+0000 7fd0a5ccb640 1 Processor -- start 2024-09-21T09:43:08.761 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.759+0000 7fd0a5ccb640 1 -- start start 2024-09-21T09:43:08.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.760+0000 7fd0a5ccb640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd0a00715f0 0x7fd0a0121db0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:08.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.760+0000 7fd0a5ccb640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd0a0071f10 0x7fd0a01222f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:08.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.760+0000 7fd0a5ccb640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd0a0123090 0x7fd0a012fd80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:08.764 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.760+0000 7fd0a5ccb640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fd0a0134480 con 0x7fd0a00715f0 2024-09-21T09:43:08.764 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.760+0000 7fd0a5ccb640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fd0a0134300 con 0x7fd0a0123090 2024-09-21T09:43:08.764 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.760+0000 7fd0a5ccb640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fd0a0134600 con 0x7fd0a0071f10 2024-09-21T09:43:08.764 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.760+0000 7fd09f7fe640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd0a00715f0 0x7fd0a0121db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:08.764 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.760+0000 7fd09effd640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd0a0071f10 0x7fd0a01222f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:08.764 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.760+0000 7fd09f7fe640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd0a00715f0 0x7fd0a0121db0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:53546/0 (socket says 172.21.15.92:53546) 2024-09-21T09:43:08.764 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.760+0000 7fd09f7fe640 1 -- 172.21.15.92:0/2568507335 learned_addr learned my addr 172.21.15.92:0/2568507335 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:08.764 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.760+0000 7fd09ffff640 1 --2- 172.21.15.92:0/2568507335 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd0a0123090 0x7fd0a012fd80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:08.764 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.761+0000 7fd09effd640 1 -- 172.21.15.92:0/2568507335 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd0a0123090 msgr2=0x7fd0a012fd80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:08.764 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.761+0000 7fd09effd640 1 --2- 172.21.15.92:0/2568507335 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd0a0123090 0x7fd0a012fd80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.764 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.761+0000 7fd09effd640 1 -- 172.21.15.92:0/2568507335 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd0a00715f0 msgr2=0x7fd0a0121db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:08.764 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.761+0000 7fd09effd640 1 --2- 172.21.15.92:0/2568507335 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd0a00715f0 0x7fd0a0121db0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.764 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.761+0000 7fd09effd640 1 -- 172.21.15.92:0/2568507335 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd0a01321d0 con 0x7fd0a0071f10 2024-09-21T09:43:08.765 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.761+0000 7fd09f7fe640 1 --2- 172.21.15.92:0/2568507335 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd0a00715f0 0x7fd0a0121db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:08.765 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.761+0000 7fd09effd640 1 --2- 172.21.15.92:0/2568507335 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd0a0071f10 0x7fd0a01222f0 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7fd09400e9f0 tx=0x7fd09400edf0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:08.765 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.761+0000 7fd087fff640 1 -- 172.21.15.92:0/2568507335 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd09400ccc0 con 0x7fd0a0071f10 2024-09-21T09:43:08.765 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.761+0000 7fd087fff640 1 -- 172.21.15.92:0/2568507335 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fd094004020 con 0x7fd0a0071f10 2024-09-21T09:43:08.765 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.761+0000 7fd0a5ccb640 1 -- 172.21.15.92:0/2568507335 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd0a0126ee0 con 0x7fd0a0071f10 2024-09-21T09:43:08.765 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.761+0000 7fd087fff640 1 -- 172.21.15.92:0/2568507335 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd09400ce60 con 0x7fd0a0071f10 2024-09-21T09:43:08.765 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.761+0000 7fd0a5ccb640 1 -- 172.21.15.92:0/2568507335 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fd0a0122830 con 0x7fd0a0071f10 2024-09-21T09:43:08.765 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.763+0000 7fd0a5ccb640 1 -- 172.21.15.92:0/2568507335 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd0a009b9a0 con 0x7fd0a0071f10 2024-09-21T09:43:08.766 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.763+0000 7fd087fff640 1 -- 172.21.15.92:0/2568507335 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 8) ==== 63442+0+0 (secure 0 0 0) 0x7fd0940035f0 con 0x7fd0a0071f10 2024-09-21T09:43:08.766 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.763+0000 7fd087fff640 1 -- 172.21.15.92:0/2568507335 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7fd0940697a0 con 0x7fd0a0071f10 2024-09-21T09:43:08.769 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.767+0000 7fd087fff640 1 -- 172.21.15.92:0/2568507335 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fd0a009b9a0 con 0x7fd0a0071f10 2024-09-21T09:43:08.804 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.802+0000 7fb4d50bd640 1 -- 172.21.15.92:0/551467144 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fb4d012cd60 msgr2=0x7fb4d012d1d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:08.804 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.802+0000 7fb4d50bd640 1 --2- 172.21.15.92:0/551467144 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fb4d012cd60 0x7fb4d012d1d0 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7fb4c8005fd0 tx=0x7fb4c802a8f0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.804 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.802+0000 7fb4d50bd640 1 -- 172.21.15.92:0/551467144 shutdown_connections 2024-09-21T09:43:08.804 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.802+0000 7fb4d50bd640 1 --2- 172.21.15.92:0/551467144 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fb4d012cd60 0x7fb4d012d1d0 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.805 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.802+0000 7fb4d50bd640 1 --2- 172.21.15.92:0/551467144 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fb4d0129e30 0x7fb4d012c820 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.805 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.802+0000 7fb4d50bd640 1 --2- 172.21.15.92:0/551467144 >> v2:172.21.15.92:3300/0 conn(0x7fb4d0129510 0x7fb4d01298f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.805 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.802+0000 7fb4d50bd640 1 -- 172.21.15.92:0/551467144 >> 172.21.15.92:0/551467144 conn(0x7fb4d009d370 msgr2=0x7fb4d009d780 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:08.805 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.802+0000 7fb4d50bd640 1 -- 172.21.15.92:0/551467144 shutdown_connections 2024-09-21T09:43:08.805 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.802+0000 7fb4d50bd640 1 -- 172.21.15.92:0/551467144 wait complete. 2024-09-21T09:43:08.807 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.803+0000 7fb4d50bd640 1 Processor -- start 2024-09-21T09:43:08.807 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.803+0000 7fb4d50bd640 1 -- start start 2024-09-21T09:43:08.807 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.804+0000 7fb4d50bd640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fb4d0129510 0x7fb4d0122320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:08.807 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.804+0000 7fb4d50bd640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fb4d0129e30 0x7fb4d0122860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:08.807 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.804+0000 7fb4d50bd640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fb4d012cd60 0x7fb4d0130850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:08.807 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.804+0000 7fb4d50bd640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fb4d0134580 con 0x7fb4d0129e30 2024-09-21T09:43:08.807 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.804+0000 7fb4d50bd640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fb4d0134400 con 0x7fb4d012cd60 2024-09-21T09:43:08.807 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.804+0000 7fb4d50bd640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fb4d0134700 con 0x7fb4d0129510 2024-09-21T09:43:08.807 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.804+0000 7fb4ced9d640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fb4d0129510 0x7fb4d0122320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:08.808 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.804+0000 7fb4cf59e640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fb4d012cd60 0x7fb4d0130850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:08.808 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.804+0000 7fb4ced9d640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fb4d0129510 0x7fb4d0122320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:49296/0 (socket says 172.21.15.92:49296) 2024-09-21T09:43:08.808 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.804+0000 7fb4ced9d640 1 -- 172.21.15.92:0/952667289 learned_addr learned my addr 172.21.15.92:0/952667289 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:08.808 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.804+0000 7fb4ce59c640 1 --2- 172.21.15.92:0/952667289 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fb4d0129e30 0x7fb4d0122860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:08.808 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.805+0000 7fb4cf59e640 1 -- 172.21.15.92:0/952667289 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fb4d0129510 msgr2=0x7fb4d0122320 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:08.808 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.805+0000 7fb4cf59e640 1 --2- 172.21.15.92:0/952667289 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fb4d0129510 0x7fb4d0122320 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.808 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.805+0000 7fb4cf59e640 1 -- 172.21.15.92:0/952667289 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fb4d0129e30 msgr2=0x7fb4d0122860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:08.808 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.805+0000 7fb4cf59e640 1 --2- 172.21.15.92:0/952667289 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fb4d0129e30 0x7fb4d0122860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.808 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.805+0000 7fb4cf59e640 1 -- 172.21.15.92:0/952667289 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb4d01327d0 con 0x7fb4d012cd60 2024-09-21T09:43:08.808 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.805+0000 7fb4ced9d640 1 --2- 172.21.15.92:0/952667289 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fb4d0129510 0x7fb4d0122320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:08.808 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.805+0000 7fb4cf59e640 1 --2- 172.21.15.92:0/952667289 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fb4d012cd60 0x7fb4d0130850 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7fb4d0073690 tx=0x7fb4c8005dc0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:08.808 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.805+0000 7fb4b77fe640 1 -- 172.21.15.92:0/952667289 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb4c8040820 con 0x7fb4d012cd60 2024-09-21T09:43:08.808 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.805+0000 7fb4b77fe640 1 -- 172.21.15.92:0/952667289 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fb4c802d070 con 0x7fb4d012cd60 2024-09-21T09:43:08.808 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.805+0000 7fb4b77fe640 1 -- 172.21.15.92:0/952667289 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb4c8040c40 con 0x7fb4d012cd60 2024-09-21T09:43:08.809 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.805+0000 7fb4d50bd640 1 -- 172.21.15.92:0/952667289 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb4d0115d30 con 0x7fb4d012cd60 2024-09-21T09:43:08.809 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.805+0000 7fb4d50bd640 1 -- 172.21.15.92:0/952667289 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fb4d01331d0 con 0x7fb4d012cd60 2024-09-21T09:43:08.809 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.806+0000 7fb4d50bd640 1 -- 172.21.15.92:0/952667289 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb4d012c820 con 0x7fb4d012cd60 2024-09-21T09:43:08.809 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.807+0000 7fb4b77fe640 1 -- 172.21.15.92:0/952667289 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 8) ==== 63442+0+0 (secure 0 0 0) 0x7fb4c8004040 con 0x7fb4d012cd60 2024-09-21T09:43:08.809 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.807+0000 7fb4b77fe640 1 -- 172.21.15.92:0/952667289 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7fb4c808eb70 con 0x7fb4d012cd60 2024-09-21T09:43:08.816 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.814+0000 7fb4b77fe640 1 -- 172.21.15.92:0/952667289 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fb4d01327d0 con 0x7fb4d012cd60 2024-09-21T09:43:08.850 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.848+0000 7f262db69640 1 -- 172.21.15.92:0/5423290 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2628099830 msgr2=0x7f2628133dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:08.850 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.848+0000 7f262db69640 1 --2- 172.21.15.92:0/5423290 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2628099830 0x7f2628133dd0 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f261c009830 tx=0x7f261c02a3f0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.850 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.848+0000 7f262db69640 1 -- 172.21.15.92:0/5423290 shutdown_connections 2024-09-21T09:43:08.851 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.848+0000 7f262db69640 1 --2- 172.21.15.92:0/5423290 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2628134310 0x7f2628136700 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.851 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.848+0000 7f262db69640 1 --2- 172.21.15.92:0/5423290 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2628099830 0x7f2628133dd0 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.851 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.848+0000 7f262db69640 1 -- 172.21.15.92:0/5423290 >> 172.21.15.92:0/5423290 conn(0x7f26280a1790 msgr2=0x7f26280a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:08.851 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.848+0000 7f262db69640 1 -- 172.21.15.92:0/5423290 shutdown_connections 2024-09-21T09:43:08.851 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.848+0000 7f262db69640 1 -- 172.21.15.92:0/5423290 wait complete. 2024-09-21T09:43:08.852 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.849+0000 7f262db69640 1 Processor -- start 2024-09-21T09:43:08.852 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.849+0000 7f262db69640 1 -- start start 2024-09-21T09:43:08.852 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.850+0000 7f262db69640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2628098d40 0x7f26281c4e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:08.852 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.850+0000 7f262db69640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2628099830 0x7f26281c53b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:08.852 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.850+0000 7f262db69640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2628134310 0x7f262812fba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:08.852 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.850+0000 7f262db69640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f2628137890 con 0x7f2628099830 2024-09-21T09:43:08.852 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.850+0000 7f262db69640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f2628137710 con 0x7f2628134310 2024-09-21T09:43:08.852 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.850+0000 7f262db69640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f2628137a10 con 0x7f2628098d40 2024-09-21T09:43:08.852 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.850+0000 7f26277fe640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2628098d40 0x7f26281c4e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:08.852 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.850+0000 7f26277fe640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2628098d40 0x7f26281c4e70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:49316/0 (socket says 172.21.15.92:49316) 2024-09-21T09:43:08.853 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.850+0000 7f26277fe640 1 -- 172.21.15.92:0/4219672040 learned_addr learned my addr 172.21.15.92:0/4219672040 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:08.853 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.850+0000 7f2626ffd640 1 --2- 172.21.15.92:0/4219672040 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2628099830 0x7f26281c53b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:08.853 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.850+0000 7f2627fff640 1 --2- 172.21.15.92:0/4219672040 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2628134310 0x7f262812fba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:08.854 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.850+0000 7f2627fff640 1 -- 172.21.15.92:0/4219672040 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2628098d40 msgr2=0x7f26281c4e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:08.854 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.850+0000 7f2627fff640 1 --2- 172.21.15.92:0/4219672040 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2628098d40 0x7f26281c4e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.854 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.850+0000 7f2627fff640 1 -- 172.21.15.92:0/4219672040 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2628099830 msgr2=0x7f26281c53b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:08.854 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.850+0000 7f2627fff640 1 --2- 172.21.15.92:0/4219672040 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2628099830 0x7f26281c53b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.854 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.850+0000 7f2627fff640 1 -- 172.21.15.92:0/4219672040 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2628131eb0 con 0x7f2628134310 2024-09-21T09:43:08.854 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.851+0000 7f2627fff640 1 --2- 172.21.15.92:0/4219672040 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2628134310 0x7f262812fba0 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f2620005fd0 tx=0x7f262000c740 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:08.854 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.851+0000 7f260ffff640 1 -- 172.21.15.92:0/4219672040 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2620004b20 con 0x7f2628134310 2024-09-21T09:43:08.854 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.851+0000 7f260ffff640 1 -- 172.21.15.92:0/4219672040 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2620004db0 con 0x7f2628134310 2024-09-21T09:43:08.854 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.851+0000 7f262db69640 1 -- 172.21.15.92:0/4219672040 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f26281c58f0 con 0x7f2628134310 2024-09-21T09:43:08.855 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.851+0000 7f260ffff640 1 -- 172.21.15.92:0/4219672040 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2620003bd0 con 0x7f2628134310 2024-09-21T09:43:08.855 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.851+0000 7f262db69640 1 -- 172.21.15.92:0/4219672040 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f2628132570 con 0x7f2628134310 2024-09-21T09:43:08.855 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.852+0000 7f260ffff640 1 -- 172.21.15.92:0/4219672040 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 8) ==== 63442+0+0 (secure 0 0 0) 0x7f262001d020 con 0x7f2628134310 2024-09-21T09:43:08.855 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.852+0000 7f262db69640 1 -- 172.21.15.92:0/4219672040 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f262809d0e0 con 0x7f2628134310 2024-09-21T09:43:08.855 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.853+0000 7f260ffff640 1 -- 172.21.15.92:0/4219672040 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7f26200695a0 con 0x7f2628134310 2024-09-21T09:43:08.861 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.860+0000 7f260ffff640 1 -- 172.21.15.92:0/4219672040 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f262809d0e0 con 0x7f2628134310 2024-09-21T09:43:08.871 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.869+0000 7f6a68fe7640 1 -- 172.21.15.92:0/2339857453 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6a6412ea40 msgr2=0x7f6a6412f0e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:08.871 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.869+0000 7f6a68fe7640 1 --2- 172.21.15.92:0/2339857453 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6a6412ea40 0x7f6a6412f0e0 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f6a54009860 tx=0x7f6a5402a3d0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.871 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.869+0000 7f6a68fe7640 1 -- 172.21.15.92:0/2339857453 shutdown_connections 2024-09-21T09:43:08.871 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.869+0000 7f6a68fe7640 1 --2- 172.21.15.92:0/2339857453 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6a64099480 0x7f6a640998f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.871 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.869+0000 7f6a68fe7640 1 --2- 172.21.15.92:0/2339857453 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6a6412ea40 0x7f6a6412f0e0 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.871 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.869+0000 7f6a68fe7640 1 -- 172.21.15.92:0/2339857453 >> 172.21.15.92:0/2339857453 conn(0x7f6a640a1790 msgr2=0x7f6a640a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:08.871 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.869+0000 7f6a68fe7640 1 -- 172.21.15.92:0/2339857453 shutdown_connections 2024-09-21T09:43:08.872 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.869+0000 7f6a68fe7640 1 -- 172.21.15.92:0/2339857453 wait complete. 2024-09-21T09:43:08.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.870+0000 7f6a68fe7640 1 Processor -- start 2024-09-21T09:43:08.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.870+0000 7f6a68fe7640 1 -- start start 2024-09-21T09:43:08.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.871+0000 7f6a68fe7640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6a64099480 0x7f6a64071570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:08.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.871+0000 7f6a68fe7640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6a6412ea40 0x7f6a64071ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:08.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.871+0000 7f6a68fe7640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6a6412f620 0x7f6a64071ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:08.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.871+0000 7f6a68fe7640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f6a64137a50 con 0x7f6a64099480 2024-09-21T09:43:08.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.871+0000 7f6a68fe7640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f6a641378d0 con 0x7f6a6412ea40 2024-09-21T09:43:08.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.871+0000 7f6a68fe7640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f6a64137bd0 con 0x7f6a6412f620 2024-09-21T09:43:08.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.871+0000 7f6a61d9b640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6a6412ea40 0x7f6a64071ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:08.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.871+0000 7f6a6259c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6a64099480 0x7f6a64071570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:08.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.871+0000 7f6a61d9b640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6a6412ea40 0x7f6a64071ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:59594/0 (socket says 172.21.15.92:59594) 2024-09-21T09:43:08.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.871+0000 7f6a61d9b640 1 -- 172.21.15.92:0/3100352046 learned_addr learned my addr 172.21.15.92:0/3100352046 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:08.875 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.871+0000 7f6a62d9d640 1 --2- 172.21.15.92:0/3100352046 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6a6412f620 0x7f6a64071ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:08.875 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.871+0000 7f6a6259c640 1 -- 172.21.15.92:0/3100352046 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6a6412f620 msgr2=0x7f6a64071ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:08.875 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.871+0000 7f6a6259c640 1 --2- 172.21.15.92:0/3100352046 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6a6412f620 0x7f6a64071ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.875 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.871+0000 7f6a6259c640 1 -- 172.21.15.92:0/3100352046 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6a6412ea40 msgr2=0x7f6a64071ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:08.875 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.871+0000 7f6a6259c640 1 --2- 172.21.15.92:0/3100352046 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6a6412ea40 0x7f6a64071ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.875 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.871+0000 7f6a6259c640 1 -- 172.21.15.92:0/3100352046 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6a64073280 con 0x7f6a64099480 2024-09-21T09:43:08.875 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.871+0000 7f6a61d9b640 1 --2- 172.21.15.92:0/3100352046 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6a6412ea40 0x7f6a64071ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:08.875 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.871+0000 7f6a62d9d640 1 --2- 172.21.15.92:0/3100352046 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6a6412f620 0x7f6a64071ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:08.875 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.872+0000 7f6a6259c640 1 --2- 172.21.15.92:0/3100352046 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6a64099480 0x7f6a64071570 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f6a54031000 tx=0x7f6a54032b90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:08.875 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.872+0000 7f6a52ffd640 1 -- 172.21.15.92:0/3100352046 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6a54040070 con 0x7f6a64099480 2024-09-21T09:43:08.875 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.872+0000 7f6a52ffd640 1 -- 172.21.15.92:0/3100352046 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f6a540038e0 con 0x7f6a64099480 2024-09-21T09:43:08.875 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.872+0000 7f6a52ffd640 1 -- 172.21.15.92:0/3100352046 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6a54003d10 con 0x7f6a64099480 2024-09-21T09:43:08.875 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.872+0000 7f6a68fe7640 1 -- 172.21.15.92:0/3100352046 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6a64072530 con 0x7f6a64099480 2024-09-21T09:43:08.875 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.872+0000 7f6a68fe7640 1 -- 172.21.15.92:0/3100352046 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6a641cae60 con 0x7f6a64099480 2024-09-21T09:43:08.876 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.873+0000 7f6a52ffd640 1 -- 172.21.15.92:0/3100352046 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 8) ==== 63442+0+0 (secure 0 0 0) 0x7f6a54039040 con 0x7f6a64099480 2024-09-21T09:43:08.876 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.873+0000 7f6a68fe7640 1 -- 172.21.15.92:0/3100352046 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6a64128580 con 0x7f6a64099480 2024-09-21T09:43:08.876 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.873+0000 7f6a52ffd640 1 -- 172.21.15.92:0/3100352046 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7f6a5408a1d0 con 0x7f6a64099480 2024-09-21T09:43:08.878 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.877+0000 7f6a52ffd640 1 -- 172.21.15.92:0/3100352046 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f6a64128580 con 0x7f6a64099480 2024-09-21T09:43:08.978 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.976+0000 7fd0a5ccb640 1 -- 172.21.15.92:0/2568507335 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7fd0a0122af0 con 0x7fd0a0071f10 2024-09-21T09:43:08.978 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.976+0000 7fd087fff640 1 -- 172.21.15.92:0/2568507335 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fd094009390 con 0x7fd0a0071f10 2024-09-21T09:43:08.978 INFO:teuthology.orchestra.run.smithi092.stdout:47244640277 2024-09-21T09:43:08.980 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.978+0000 7fd0a5ccb640 1 -- 172.21.15.92:0/2568507335 >> v1:172.21.15.92:6800/2299163855 conn(0x7fd08c051450 legacy=0x7fd08c053910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:08.980 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.978+0000 7fd0a5ccb640 1 -- 172.21.15.92:0/2568507335 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd0a0071f10 msgr2=0x7fd0a01222f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:08.981 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.978+0000 7fd0a5ccb640 1 --2- 172.21.15.92:0/2568507335 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd0a0071f10 0x7fd0a01222f0 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7fd09400e9f0 tx=0x7fd09400edf0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.981 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.978+0000 7fd0a5ccb640 1 -- 172.21.15.92:0/2568507335 shutdown_connections 2024-09-21T09:43:08.981 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.978+0000 7fd0a5ccb640 1 --2- 172.21.15.92:0/2568507335 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd0a0123090 0x7fd0a012fd80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.981 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.978+0000 7fd0a5ccb640 1 --2- 172.21.15.92:0/2568507335 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd0a0071f10 0x7fd0a01222f0 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.981 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.978+0000 7fd0a5ccb640 1 --2- 172.21.15.92:0/2568507335 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd0a00715f0 0x7fd0a0121db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.981 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.978+0000 7fd0a5ccb640 1 -- 172.21.15.92:0/2568507335 >> 172.21.15.92:0/2568507335 conn(0x7fd0a009d3a0 msgr2=0x7fd0a012c580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:08.981 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.978+0000 7fd0a5ccb640 1 -- 172.21.15.92:0/2568507335 shutdown_connections 2024-09-21T09:43:08.981 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.978+0000 7fd0a5ccb640 1 -- 172.21.15.92:0/2568507335 wait complete. 2024-09-21T09:43:08.981 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.978+0000 7fb4d50bd640 1 -- 172.21.15.92:0/952667289 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7fb4d00698c0 con 0x7fb4d012cd60 2024-09-21T09:43:08.981 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.979+0000 7fb4b77fe640 1 -- 172.21.15.92:0/952667289 <== mon.1 v2:172.21.15.92:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fb4c8040de0 con 0x7fb4d012cd60 2024-09-21T09:43:08.982 INFO:teuthology.orchestra.run.smithi092.stdout:47244640277 2024-09-21T09:43:08.983 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.981+0000 7fb4d50bd640 1 -- 172.21.15.92:0/952667289 >> v1:172.21.15.92:6800/2299163855 conn(0x7fb4bc051420 legacy=0x7fb4bc0538e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:08.983 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.981+0000 7fb4d50bd640 1 -- 172.21.15.92:0/952667289 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fb4d012cd60 msgr2=0x7fb4d0130850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:08.983 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.981+0000 7fb4d50bd640 1 --2- 172.21.15.92:0/952667289 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fb4d012cd60 0x7fb4d0130850 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7fb4d0073690 tx=0x7fb4c8005dc0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.983 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.981+0000 7fb4d50bd640 1 -- 172.21.15.92:0/952667289 shutdown_connections 2024-09-21T09:43:08.983 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.981+0000 7fb4d50bd640 1 --2- 172.21.15.92:0/952667289 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fb4d012cd60 0x7fb4d0130850 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.983 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.981+0000 7fb4d50bd640 1 --2- 172.21.15.92:0/952667289 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fb4d0129e30 0x7fb4d0122860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.983 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.981+0000 7fb4d50bd640 1 --2- 172.21.15.92:0/952667289 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fb4d0129510 0x7fb4d0122320 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:08.984 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.981+0000 7fb4d50bd640 1 -- 172.21.15.92:0/952667289 >> 172.21.15.92:0/952667289 conn(0x7fb4d009d370 msgr2=0x7fb4d0128220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:08.984 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.981+0000 7fb4d50bd640 1 -- 172.21.15.92:0/952667289 shutdown_connections 2024-09-21T09:43:08.984 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:08.981+0000 7fb4d50bd640 1 -- 172.21.15.92:0/952667289 wait complete. 2024-09-21T09:43:08.992 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640276 got 47244640277 for osd.0 2024-09-21T09:43:08.992 DEBUG:teuthology.parallel:result is None 2024-09-21T09:43:08.998 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640276 got 47244640277 for osd.2 2024-09-21T09:43:08.998 DEBUG:teuthology.parallel:result is None 2024-09-21T09:43:09.041 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.040+0000 7f6a68fe7640 1 -- 172.21.15.92:0/3100352046 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7f6a6412f0e0 con 0x7f6a64099480 2024-09-21T09:43:09.042 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.040+0000 7f6a52ffd640 1 -- 172.21.15.92:0/3100352046 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f6a54088e30 con 0x7f6a64099480 2024-09-21T09:43:09.042 INFO:teuthology.orchestra.run.smithi092.stdout:38654705684 2024-09-21T09:43:09.044 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.042+0000 7f6a68fe7640 1 -- 172.21.15.92:0/3100352046 >> v1:172.21.15.92:6800/2299163855 conn(0x7f6a44051480 legacy=0x7f6a44053940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:09.044 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.042+0000 7f6a68fe7640 1 -- 172.21.15.92:0/3100352046 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6a64099480 msgr2=0x7f6a64071570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:09.045 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.042+0000 7f6a68fe7640 1 --2- 172.21.15.92:0/3100352046 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6a64099480 0x7f6a64071570 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f6a54031000 tx=0x7f6a54032b90 comp rx=0 tx=0).stop 2024-09-21T09:43:09.045 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.042+0000 7f6a68fe7640 1 -- 172.21.15.92:0/3100352046 shutdown_connections 2024-09-21T09:43:09.045 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.042+0000 7f6a68fe7640 1 --2- 172.21.15.92:0/3100352046 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6a6412f620 0x7f6a64071ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.045 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.042+0000 7f6a68fe7640 1 --2- 172.21.15.92:0/3100352046 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6a6412ea40 0x7f6a64071ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.045 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.042+0000 7f6a68fe7640 1 --2- 172.21.15.92:0/3100352046 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6a64099480 0x7f6a64071570 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.045 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.042+0000 7f6a68fe7640 1 -- 172.21.15.92:0/3100352046 >> 172.21.15.92:0/3100352046 conn(0x7f6a640a1790 msgr2=0x7f6a6412beb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:09.045 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.042+0000 7f6a68fe7640 1 -- 172.21.15.92:0/3100352046 shutdown_connections 2024-09-21T09:43:09.045 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.042+0000 7f6a68fe7640 1 -- 172.21.15.92:0/3100352046 wait complete. 2024-09-21T09:43:09.054 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705684 for osd.3 2024-09-21T09:43:09.054 DEBUG:teuthology.parallel:result is None 2024-09-21T09:43:09.064 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.062+0000 7f262db69640 1 -- 172.21.15.92:0/4219672040 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7f2628099120 con 0x7f2628134310 2024-09-21T09:43:09.064 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.062+0000 7f260ffff640 1 -- 172.21.15.92:0/4219672040 <== mon.1 v2:172.21.15.92:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f2620003d70 con 0x7f2628134310 2024-09-21T09:43:09.064 INFO:teuthology.orchestra.run.smithi092.stdout:47244640275 2024-09-21T09:43:09.066 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.064+0000 7f262db69640 1 -- 172.21.15.92:0/4219672040 >> v1:172.21.15.92:6800/2299163855 conn(0x7f2614051420 legacy=0x7f26140538e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:09.066 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.064+0000 7f262db69640 1 -- 172.21.15.92:0/4219672040 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2628134310 msgr2=0x7f262812fba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:09.067 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.064+0000 7f262db69640 1 --2- 172.21.15.92:0/4219672040 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2628134310 0x7f262812fba0 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f2620005fd0 tx=0x7f262000c740 comp rx=0 tx=0).stop 2024-09-21T09:43:09.067 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.064+0000 7f262db69640 1 -- 172.21.15.92:0/4219672040 shutdown_connections 2024-09-21T09:43:09.067 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.064+0000 7f262db69640 1 --2- 172.21.15.92:0/4219672040 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2628134310 0x7f262812fba0 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.067 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.064+0000 7f262db69640 1 --2- 172.21.15.92:0/4219672040 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2628099830 0x7f26281c53b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.067 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.064+0000 7f262db69640 1 --2- 172.21.15.92:0/4219672040 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2628098d40 0x7f26281c4e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.067 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.064+0000 7f262db69640 1 -- 172.21.15.92:0/4219672040 >> 172.21.15.92:0/4219672040 conn(0x7f26280a1790 msgr2=0x7f2628128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:09.067 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.064+0000 7f262db69640 1 -- 172.21.15.92:0/4219672040 shutdown_connections 2024-09-21T09:43:09.067 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.064+0000 7f262db69640 1 -- 172.21.15.92:0/4219672040 wait complete. 2024-09-21T09:43:09.077 INFO:tasks.ceph.ceph_manager.ceph:need seq 47244640274 got 47244640275 for osd.1 2024-09-21T09:43:09.077 DEBUG:teuthology.parallel:result is None 2024-09-21T09:43:09.077 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean 2024-09-21T09:43:09.077 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T09:43:09.216 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.213+0000 7f1bf7fa1640 1 -- 172.21.15.92:0/208638687 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1bf01294b0 msgr2=0x7f1bf0129890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:09.216 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.214+0000 7f1bf7fa1640 1 --2- 172.21.15.92:0/208638687 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1bf01294b0 0x7f1bf0129890 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7f1be8005fd0 tx=0x7f1be802a540 comp rx=0 tx=0).stop 2024-09-21T09:43:09.216 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.214+0000 7f1bf7fa1640 1 -- 172.21.15.92:0/208638687 shutdown_connections 2024-09-21T09:43:09.216 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.214+0000 7f1bf7fa1640 1 --2- 172.21.15.92:0/208638687 >> v2:172.21.15.92:3300/0 conn(0x7f1bf012ced0 0x7f1bf0134b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.216 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.214+0000 7f1bf7fa1640 1 --2- 172.21.15.92:0/208638687 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1bf01294b0 0x7f1bf0129890 unknown :-1 s=CLOSED pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.216 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.214+0000 7f1bf7fa1640 1 -- 172.21.15.92:0/208638687 >> 172.21.15.92:0/208638687 conn(0x7f1bf009d480 msgr2=0x7f1bf009d890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:09.216 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.214+0000 7f1bf7fa1640 1 -- 172.21.15.92:0/208638687 shutdown_connections 2024-09-21T09:43:09.216 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.214+0000 7f1bf7fa1640 1 -- 172.21.15.92:0/208638687 wait complete. 2024-09-21T09:43:09.218 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.214+0000 7f1bf7fa1640 1 Processor -- start 2024-09-21T09:43:09.218 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.215+0000 7f1bf7fa1640 1 -- start start 2024-09-21T09:43:09.219 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.215+0000 7f1bf7fa1640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1bf01294b0 0x7f1bf01305e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:09.219 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.215+0000 7f1bf7fa1640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f1bf0129dd0 0x7f1bf0130b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:09.219 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.215+0000 7f1bf7fa1640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1bf012ced0 0x7f1bf0131060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:09.219 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.215+0000 7f1bf7fa1640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f1bf01357e0 con 0x7f1bf01294b0 2024-09-21T09:43:09.219 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.215+0000 7f1bf7fa1640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f1bf0135660 con 0x7f1bf0129dd0 2024-09-21T09:43:09.219 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.215+0000 7f1bf7fa1640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f1bf0135960 con 0x7f1bf012ced0 2024-09-21T09:43:09.219 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.215+0000 7f1bf5d3d640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1bf01294b0 0x7f1bf01305e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:09.219 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.215+0000 7f1bf5d3d640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1bf01294b0 0x7f1bf01305e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:53618/0 (socket says 172.21.15.92:53618) 2024-09-21T09:43:09.219 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.215+0000 7f1bf5d3d640 1 -- 172.21.15.92:0/1625495810 learned_addr learned my addr 172.21.15.92:0/1625495810 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:09.219 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.215+0000 7f1bf553c640 1 --2- 172.21.15.92:0/1625495810 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f1bf0129dd0 0x7f1bf0130b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:09.219 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.215+0000 7f1bf653e640 1 --2- 172.21.15.92:0/1625495810 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1bf012ced0 0x7f1bf0131060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:09.219 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.215+0000 7f1bf5d3d640 1 -- 172.21.15.92:0/1625495810 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1bf012ced0 msgr2=0x7f1bf0131060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:09.219 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.215+0000 7f1bf5d3d640 1 --2- 172.21.15.92:0/1625495810 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1bf012ced0 0x7f1bf0131060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.220 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.215+0000 7f1bf5d3d640 1 -- 172.21.15.92:0/1625495810 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f1bf0129dd0 msgr2=0x7f1bf0130b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:09.220 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.215+0000 7f1bf5d3d640 1 --2- 172.21.15.92:0/1625495810 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f1bf0129dd0 0x7f1bf0130b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.220 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.215+0000 7f1bf5d3d640 1 -- 172.21.15.92:0/1625495810 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1bf0126ef0 con 0x7f1bf01294b0 2024-09-21T09:43:09.220 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.215+0000 7f1bf653e640 1 --2- 172.21.15.92:0/1625495810 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1bf012ced0 0x7f1bf0131060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:09.220 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.215+0000 7f1bf5d3d640 1 --2- 172.21.15.92:0/1625495810 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1bf01294b0 0x7f1bf01305e0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f1be8009850 tx=0x7f1be80038a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:09.220 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.216+0000 7f1bee7fc640 1 -- 172.21.15.92:0/1625495810 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1be803f070 con 0x7f1bf01294b0 2024-09-21T09:43:09.220 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.216+0000 7f1bee7fc640 1 -- 172.21.15.92:0/1625495810 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f1be80047a0 con 0x7f1bf01294b0 2024-09-21T09:43:09.220 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.216+0000 7f1bf7fa1640 1 -- 172.21.15.92:0/1625495810 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1bf0131d00 con 0x7f1bf01294b0 2024-09-21T09:43:09.220 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.216+0000 7f1bee7fc640 1 -- 172.21.15.92:0/1625495810 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1be8035030 con 0x7f1bf01294b0 2024-09-21T09:43:09.220 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.216+0000 7f1bf7fa1640 1 -- 172.21.15.92:0/1625495810 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f1bf0132de0 con 0x7f1bf01294b0 2024-09-21T09:43:09.220 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.217+0000 7f1bf7fa1640 1 -- 172.21.15.92:0/1625495810 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1bf0129890 con 0x7f1bf01294b0 2024-09-21T09:43:09.220 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.217+0000 7f1bee7fc640 1 -- 172.21.15.92:0/1625495810 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 8) ==== 63442+0+0 (secure 0 0 0) 0x7f1be8042070 con 0x7f1bf01294b0 2024-09-21T09:43:09.221 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.217+0000 7f1bee7fc640 1 -- 172.21.15.92:0/1625495810 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7f1be808a010 con 0x7f1bf01294b0 2024-09-21T09:43:09.222 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.220+0000 7f1bee7fc640 1 -- 172.21.15.92:0/1625495810 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f1be8088c70 con 0x7f1bf01294b0 2024-09-21T09:43:09.371 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.370+0000 7f1bf7fa1640 1 -- 172.21.15.92:0/1625495810 --> v1:172.21.15.92:6800/2299163855 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f1bf00698c0 con 0x7f1bdc051420 2024-09-21T09:43:09.372 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.370+0000 7f1bee7fc640 1 -- 172.21.15.92:0/1625495810 <== mgr.14102 v1:172.21.15.92:6800/2299163855 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32033 (unknown 2965378022 0 2572040712) 0x7f1bf00698c0 con 0x7f1bdc051420 2024-09-21T09:43:09.373 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:09.373 INFO:teuthology.orchestra.run.smithi092.stderr:dumped all 2024-09-21T09:43:09.374 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.372+0000 7f1bf7fa1640 1 -- 172.21.15.92:0/1625495810 >> v1:172.21.15.92:6800/2299163855 conn(0x7f1bdc051420 legacy=0x7f1bdc0538e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:09.374 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.372+0000 7f1bf7fa1640 1 -- 172.21.15.92:0/1625495810 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1bf01294b0 msgr2=0x7f1bf01305e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:09.374 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.372+0000 7f1bf7fa1640 1 --2- 172.21.15.92:0/1625495810 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1bf01294b0 0x7f1bf01305e0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f1be8009850 tx=0x7f1be80038a0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.374 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.372+0000 7f1bf7fa1640 1 -- 172.21.15.92:0/1625495810 shutdown_connections 2024-09-21T09:43:09.374 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.372+0000 7f1bf7fa1640 1 --2- 172.21.15.92:0/1625495810 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1bf012ced0 0x7f1bf0131060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.375 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.372+0000 7f1bf7fa1640 1 --2- 172.21.15.92:0/1625495810 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f1bf0129dd0 0x7f1bf0130b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.375 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.372+0000 7f1bf7fa1640 1 --2- 172.21.15.92:0/1625495810 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1bf01294b0 0x7f1bf01305e0 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.375 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.372+0000 7f1bf7fa1640 1 -- 172.21.15.92:0/1625495810 >> 172.21.15.92:0/1625495810 conn(0x7f1bf009d480 msgr2=0x7f1bf009a9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:09.375 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.372+0000 7f1bf7fa1640 1 -- 172.21.15.92:0/1625495810 shutdown_connections 2024-09-21T09:43:09.375 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.372+0000 7f1bf7fa1640 1 -- 172.21.15.92:0/1625495810 wait complete. 2024-09-21T09:43:09.384 INFO:teuthology.orchestra.run.smithi092.stdout:{"pg_ready":true,"pg_map":{"version":4,"stamp":"2024-09-21T09:43:08.271926+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":109976,"kb_used_data":1856,"kb_used_omap":6,"kb_used_meta":108025,"kb_avail":377377384,"statfs":{"total":386547056640,"available":386434441216,"internally_reserved":0,"allocated":1900544,"data_stored":1383130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6375,"internal_metadata":110618393},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"1.009752"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-21T09:43:06.262934+0000","last_change":"2024-09-21T09:41:55.860096+0000","last_active":"2024-09-21T09:43:06.262934+0000","last_peered":"2024-09-21T09:43:06.262934+0000","last_clean":"2024-09-21T09:43:06.262934+0000","last_became_active":"2024-09-21T09:41:53.886716+0000","last_became_peered":"2024-09-21T09:41:53.886716+0000","last_unstale":"2024-09-21T09:43:06.262934+0000","last_undegraded":"2024-09-21T09:43:06.262934+0000","last_fullsized":"2024-09-21T09:43:06.262934+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:44:13.256802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084231599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-21T09:43:06.320387+0000","last_change":"2024-09-21T09:41:55.863252+0000","last_active":"2024-09-21T09:43:06.320387+0000","last_peered":"2024-09-21T09:43:06.320387+0000","last_clean":"2024-09-21T09:43:06.320387+0000","last_became_active":"2024-09-21T09:41:53.887209+0000","last_became_peered":"2024-09-21T09:41:53.887209+0000","last_unstale":"2024-09-21T09:43:06.320387+0000","last_undegraded":"2024-09-21T09:43:06.320387+0000","last_fullsized":"2024-09-21T09:43:06.320387+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:43:15.259908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049204800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-21T09:43:06.262883+0000","last_change":"2024-09-21T09:41:55.860396+0000","last_active":"2024-09-21T09:43:06.262883+0000","last_peered":"2024-09-21T09:43:06.262883+0000","last_clean":"2024-09-21T09:43:06.262883+0000","last_became_active":"2024-09-21T09:41:53.886757+0000","last_became_peered":"2024-09-21T09:41:53.886757+0000","last_unstale":"2024-09-21T09:43:06.262883+0000","last_undegraded":"2024-09-21T09:43:06.262883+0000","last_fullsized":"2024-09-21T09:43:06.262883+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:49:19.741169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080402800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-21T09:43:06.320319+0000","last_change":"2024-09-21T09:41:55.863219+0000","last_active":"2024-09-21T09:43:06.320319+0000","last_peered":"2024-09-21T09:43:06.320319+0000","last_clean":"2024-09-21T09:43:06.320319+0000","last_became_active":"2024-09-21T09:41:53.886793+0000","last_became_peered":"2024-09-21T09:41:53.886793+0000","last_unstale":"2024-09-21T09:43:06.320319+0000","last_undegraded":"2024-09-21T09:43:06.320319+0000","last_fullsized":"2024-09-21T09:43:06.320319+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:42:08.531449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000465849,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"18'2","reported_seq":27,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-21T09:43:06.262809+0000","last_change":"2024-09-21T09:41:55.859682+0000","last_active":"2024-09-21T09:43:06.262809+0000","last_peered":"2024-09-21T09:43:06.262809+0000","last_clean":"2024-09-21T09:43:06.262809+0000","last_became_active":"2024-09-21T09:41:53.840867+0000","last_became_peered":"2024-09-21T09:41:53.840867+0000","last_unstale":"2024-09-21T09:43:06.262809+0000","last_undegraded":"2024-09-21T09:43:06.262809+0000","last_fullsized":"2024-09-21T09:43:06.262809+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:47:02.641520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010637209999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-21T09:43:06.317044+0000","last_change":"2024-09-21T09:41:55.858936+0000","last_active":"2024-09-21T09:43:06.317044+0000","last_peered":"2024-09-21T09:43:06.317044+0000","last_clean":"2024-09-21T09:43:06.317044+0000","last_became_active":"2024-09-21T09:41:53.846664+0000","last_became_peered":"2024-09-21T09:41:53.846664+0000","last_unstale":"2024-09-21T09:43:06.317044+0000","last_undegraded":"2024-09-21T09:43:06.317044+0000","last_fullsized":"2024-09-21T09:43:06.317044+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:05:39.687344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049819100000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-21T09:43:06.262761+0000","last_change":"2024-09-21T09:41:55.859721+0000","last_active":"2024-09-21T09:43:06.262761+0000","last_peered":"2024-09-21T09:43:06.262761+0000","last_clean":"2024-09-21T09:43:06.262761+0000","last_became_active":"2024-09-21T09:41:53.838115+0000","last_became_peered":"2024-09-21T09:41:53.838115+0000","last_unstale":"2024-09-21T09:43:06.262761+0000","last_undegraded":"2024-09-21T09:43:06.262761+0000","last_fullsized":"2024-09-21T09:43:06.262761+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:53:01.492593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088008499999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"16'1","reported_seq":26,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-21T09:43:06.262868+0000","last_change":"2024-09-21T09:41:55.860023+0000","last_active":"2024-09-21T09:43:06.262868+0000","last_peered":"2024-09-21T09:43:06.262868+0000","last_clean":"2024-09-21T09:43:06.262868+0000","last_became_active":"2024-09-21T09:41:53.843742+0000","last_became_peered":"2024-09-21T09:41:53.843742+0000","last_unstale":"2024-09-21T09:43:06.262868+0000","last_undegraded":"2024-09-21T09:43:06.262868+0000","last_fullsized":"2024-09-21T09:43:06.262868+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:14:31.505673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010430319999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"14'97","reported_seq":151,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-21T09:43:06.320264+0000","last_change":"2024-09-21T09:41:51.820265+0000","last_active":"2024-09-21T09:43:06.320264+0000","last_peered":"2024-09-21T09:43:06.320264+0000","last_clean":"2024-09-21T09:43:06.320264+0000","last_became_active":"2024-09-21T09:41:51.819950+0000","last_became_peered":"2024-09-21T09:41:51.819950+0000","last_unstale":"2024-09-21T09:43:06.320264+0000","last_undegraded":"2024-09-21T09:43:06.320264+0000","last_fullsized":"2024-09-21T09:43:06.320264+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:50.808452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:50.808452+0000","last_clean_scrub_stamp":"2024-09-21T09:41:50.808452+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:27:36.916853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":9,"seq":38654705684,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27208,"kb_used_data":180,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344632,"statfs":{"total":96636764160,"available":96608903168,"internally_reserved":0,"allocated":184320,"data_stored":52329,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":2,"up_from":11,"seq":47244640277,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27204,"kb_used_data":176,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344636,"statfs":{"total":96636764160,"available":96608907264,"internally_reserved":0,"allocated":180224,"data_stored":52310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":11,"seq":47244640275,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27784,"kb_used_data":752,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344056,"statfs":{"total":96636764160,"available":96608313344,"internally_reserved":0,"allocated":770048,"data_stored":639255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":11,"seq":47244640277,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27780,"kb_used_data":748,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344060,"statfs":{"total":96636764160,"available":96608317440,"internally_reserved":0,"allocated":765952,"data_stored":639236,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:43:09.385 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T09:43:09.485 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.483+0000 7fab07783640 1 -- 172.21.15.92:0/3070512264 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fab00131030 msgr2=0x7fab00133420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:09.485 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.483+0000 7fab07783640 1 --2- 172.21.15.92:0/3070512264 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fab00131030 0x7fab00133420 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7faaf0009be0 tx=0x7faaf002a7c0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.485 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.483+0000 7fab07783640 1 -- 172.21.15.92:0/3070512264 shutdown_connections 2024-09-21T09:43:09.485 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.483+0000 7fab07783640 1 --2- 172.21.15.92:0/3070512264 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fab00131030 0x7fab00133420 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.485 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.483+0000 7fab07783640 1 --2- 172.21.15.92:0/3070512264 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fab0012c330 0x7fab00130af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.486 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.483+0000 7fab07783640 1 -- 172.21.15.92:0/3070512264 >> 172.21.15.92:0/3070512264 conn(0x7fab0009a2e0 msgr2=0x7fab00097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:09.486 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.483+0000 7fab07783640 1 -- 172.21.15.92:0/3070512264 shutdown_connections 2024-09-21T09:43:09.486 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.483+0000 7fab07783640 1 -- 172.21.15.92:0/3070512264 wait complete. 2024-09-21T09:43:09.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.484+0000 7fab07783640 1 Processor -- start 2024-09-21T09:43:09.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.484+0000 7fab07783640 1 -- start start 2024-09-21T09:43:09.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.484+0000 7fab07783640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fab000991f0 0x7fab0008cd30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:09.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.484+0000 7fab07783640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fab0012c330 0x7fab0008d270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:09.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.484+0000 7fab07783640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fab00131030 0x7fab0008d7b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:09.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.484+0000 7fab07783640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fab001346b0 con 0x7fab000991f0 2024-09-21T09:43:09.488 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.484+0000 7fab07783640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fab00134530 con 0x7fab00131030 2024-09-21T09:43:09.489 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.484+0000 7fab07783640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fab00134830 con 0x7fab0012c330 2024-09-21T09:43:09.489 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.484+0000 7fab0551f640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fab000991f0 0x7fab0008cd30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:09.489 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.484+0000 7fab04d1e640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fab0012c330 0x7fab0008d270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:09.489 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.484+0000 7fab0551f640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fab000991f0 0x7fab0008cd30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:53642/0 (socket says 172.21.15.92:53642) 2024-09-21T09:43:09.489 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.484+0000 7fab0551f640 1 -- 172.21.15.92:0/4219905050 learned_addr learned my addr 172.21.15.92:0/4219905050 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:09.489 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.484+0000 7fab04d1e640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fab0012c330 0x7fab0008d270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:49380/0 (socket says 172.21.15.92:49380) 2024-09-21T09:43:09.489 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.484+0000 7fab05d20640 1 --2- 172.21.15.92:0/4219905050 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fab00131030 0x7fab0008d7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:09.489 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.485+0000 7fab0551f640 1 -- 172.21.15.92:0/4219905050 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fab0012c330 msgr2=0x7fab0008d270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:09.489 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.485+0000 7fab0551f640 1 --2- 172.21.15.92:0/4219905050 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fab0012c330 0x7fab0008d270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.489 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.485+0000 7fab0551f640 1 -- 172.21.15.92:0/4219905050 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fab00131030 msgr2=0x7fab0008d7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:09.489 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.485+0000 7fab0551f640 1 --2- 172.21.15.92:0/4219905050 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fab00131030 0x7fab0008d7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.489 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.485+0000 7fab0551f640 1 -- 172.21.15.92:0/4219905050 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fab00094030 con 0x7fab000991f0 2024-09-21T09:43:09.489 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.485+0000 7fab0551f640 1 --2- 172.21.15.92:0/4219905050 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fab000991f0 0x7fab0008cd30 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7faaf800c400 tx=0x7faaf800c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:09.489 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.485+0000 7faafdffb640 1 -- 172.21.15.92:0/4219905050 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faaf8011790 con 0x7fab000991f0 2024-09-21T09:43:09.490 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.485+0000 7faafdffb640 1 -- 172.21.15.92:0/4219905050 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7faaf8017070 con 0x7fab000991f0 2024-09-21T09:43:09.490 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.485+0000 7faafdffb640 1 -- 172.21.15.92:0/4219905050 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7faaf8011ca0 con 0x7fab000991f0 2024-09-21T09:43:09.490 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.485+0000 7fab07783640 1 -- 172.21.15.92:0/4219905050 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fab000903a0 con 0x7fab000991f0 2024-09-21T09:43:09.490 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.485+0000 7fab07783640 1 -- 172.21.15.92:0/4219905050 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fab0008f530 con 0x7fab000991f0 2024-09-21T09:43:09.490 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.486+0000 7faafdffb640 1 -- 172.21.15.92:0/4219905050 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 8) ==== 63442+0+0 (secure 0 0 0) 0x7faaf801c460 con 0x7fab000991f0 2024-09-21T09:43:09.490 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.486+0000 7fab07783640 1 -- 172.21.15.92:0/4219905050 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fab0012bc20 con 0x7fab000991f0 2024-09-21T09:43:09.490 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.486+0000 7faafdffb640 1 -- 172.21.15.92:0/4219905050 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7faaf8068cb0 con 0x7fab000991f0 2024-09-21T09:43:09.491 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.490+0000 7faafdffb640 1 -- 172.21.15.92:0/4219905050 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7faaf8067dd0 con 0x7fab000991f0 2024-09-21T09:43:09.637 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.635+0000 7fab07783640 1 -- 172.21.15.92:0/4219905050 --> v1:172.21.15.92:6800/2299163855 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7fab000699c0 con 0x7faaec051420 2024-09-21T09:43:09.637 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.636+0000 7faafdffb640 1 -- 172.21.15.92:0/4219905050 <== mgr.14102 v1:172.21.15.92:6800/2299163855 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32033 (unknown 2965378022 0 2572040712) 0x7fab000699c0 con 0x7faaec051420 2024-09-21T09:43:09.638 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:09.638 INFO:teuthology.orchestra.run.smithi092.stderr:dumped all 2024-09-21T09:43:09.640 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.638+0000 7fab07783640 1 -- 172.21.15.92:0/4219905050 >> v1:172.21.15.92:6800/2299163855 conn(0x7faaec051420 legacy=0x7faaec0538e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:09.640 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.638+0000 7fab07783640 1 -- 172.21.15.92:0/4219905050 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fab000991f0 msgr2=0x7fab0008cd30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:09.640 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.638+0000 7fab07783640 1 --2- 172.21.15.92:0/4219905050 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fab000991f0 0x7fab0008cd30 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7faaf800c400 tx=0x7faaf800c8c0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.640 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.638+0000 7fab07783640 1 -- 172.21.15.92:0/4219905050 shutdown_connections 2024-09-21T09:43:09.640 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.638+0000 7fab07783640 1 --2- 172.21.15.92:0/4219905050 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fab00131030 0x7fab0008d7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.640 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.638+0000 7fab07783640 1 --2- 172.21.15.92:0/4219905050 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fab0012c330 0x7fab0008d270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.640 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.638+0000 7fab07783640 1 --2- 172.21.15.92:0/4219905050 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fab000991f0 0x7fab0008cd30 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.640 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.638+0000 7fab07783640 1 -- 172.21.15.92:0/4219905050 >> 172.21.15.92:0/4219905050 conn(0x7fab0009a2e0 msgr2=0x7fab000945d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:09.641 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.638+0000 7fab07783640 1 -- 172.21.15.92:0/4219905050 shutdown_connections 2024-09-21T09:43:09.641 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.638+0000 7fab07783640 1 -- 172.21.15.92:0/4219905050 wait complete. 2024-09-21T09:43:09.650 INFO:teuthology.orchestra.run.smithi092.stdout:{"pg_ready":true,"pg_map":{"version":4,"stamp":"2024-09-21T09:43:08.271926+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":109976,"kb_used_data":1856,"kb_used_omap":6,"kb_used_meta":108025,"kb_avail":377377384,"statfs":{"total":386547056640,"available":386434441216,"internally_reserved":0,"allocated":1900544,"data_stored":1383130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6375,"internal_metadata":110618393},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"1.009752"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-21T09:43:06.262934+0000","last_change":"2024-09-21T09:41:55.860096+0000","last_active":"2024-09-21T09:43:06.262934+0000","last_peered":"2024-09-21T09:43:06.262934+0000","last_clean":"2024-09-21T09:43:06.262934+0000","last_became_active":"2024-09-21T09:41:53.886716+0000","last_became_peered":"2024-09-21T09:41:53.886716+0000","last_unstale":"2024-09-21T09:43:06.262934+0000","last_undegraded":"2024-09-21T09:43:06.262934+0000","last_fullsized":"2024-09-21T09:43:06.262934+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:44:13.256802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084231599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-21T09:43:06.320387+0000","last_change":"2024-09-21T09:41:55.863252+0000","last_active":"2024-09-21T09:43:06.320387+0000","last_peered":"2024-09-21T09:43:06.320387+0000","last_clean":"2024-09-21T09:43:06.320387+0000","last_became_active":"2024-09-21T09:41:53.887209+0000","last_became_peered":"2024-09-21T09:41:53.887209+0000","last_unstale":"2024-09-21T09:43:06.320387+0000","last_undegraded":"2024-09-21T09:43:06.320387+0000","last_fullsized":"2024-09-21T09:43:06.320387+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:43:15.259908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049204800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-21T09:43:06.262883+0000","last_change":"2024-09-21T09:41:55.860396+0000","last_active":"2024-09-21T09:43:06.262883+0000","last_peered":"2024-09-21T09:43:06.262883+0000","last_clean":"2024-09-21T09:43:06.262883+0000","last_became_active":"2024-09-21T09:41:53.886757+0000","last_became_peered":"2024-09-21T09:41:53.886757+0000","last_unstale":"2024-09-21T09:43:06.262883+0000","last_undegraded":"2024-09-21T09:43:06.262883+0000","last_fullsized":"2024-09-21T09:43:06.262883+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:49:19.741169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080402800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-21T09:43:06.320319+0000","last_change":"2024-09-21T09:41:55.863219+0000","last_active":"2024-09-21T09:43:06.320319+0000","last_peered":"2024-09-21T09:43:06.320319+0000","last_clean":"2024-09-21T09:43:06.320319+0000","last_became_active":"2024-09-21T09:41:53.886793+0000","last_became_peered":"2024-09-21T09:41:53.886793+0000","last_unstale":"2024-09-21T09:43:06.320319+0000","last_undegraded":"2024-09-21T09:43:06.320319+0000","last_fullsized":"2024-09-21T09:43:06.320319+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:42:08.531449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000465849,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"18'2","reported_seq":27,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-21T09:43:06.262809+0000","last_change":"2024-09-21T09:41:55.859682+0000","last_active":"2024-09-21T09:43:06.262809+0000","last_peered":"2024-09-21T09:43:06.262809+0000","last_clean":"2024-09-21T09:43:06.262809+0000","last_became_active":"2024-09-21T09:41:53.840867+0000","last_became_peered":"2024-09-21T09:41:53.840867+0000","last_unstale":"2024-09-21T09:43:06.262809+0000","last_undegraded":"2024-09-21T09:43:06.262809+0000","last_fullsized":"2024-09-21T09:43:06.262809+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:47:02.641520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010637209999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-21T09:43:06.317044+0000","last_change":"2024-09-21T09:41:55.858936+0000","last_active":"2024-09-21T09:43:06.317044+0000","last_peered":"2024-09-21T09:43:06.317044+0000","last_clean":"2024-09-21T09:43:06.317044+0000","last_became_active":"2024-09-21T09:41:53.846664+0000","last_became_peered":"2024-09-21T09:41:53.846664+0000","last_unstale":"2024-09-21T09:43:06.317044+0000","last_undegraded":"2024-09-21T09:43:06.317044+0000","last_fullsized":"2024-09-21T09:43:06.317044+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:05:39.687344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049819100000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":25,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-21T09:43:06.262761+0000","last_change":"2024-09-21T09:41:55.859721+0000","last_active":"2024-09-21T09:43:06.262761+0000","last_peered":"2024-09-21T09:43:06.262761+0000","last_clean":"2024-09-21T09:43:06.262761+0000","last_became_active":"2024-09-21T09:41:53.838115+0000","last_became_peered":"2024-09-21T09:41:53.838115+0000","last_unstale":"2024-09-21T09:43:06.262761+0000","last_undegraded":"2024-09-21T09:43:06.262761+0000","last_fullsized":"2024-09-21T09:43:06.262761+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:53:01.492593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088008499999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"16'1","reported_seq":26,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-21T09:43:06.262868+0000","last_change":"2024-09-21T09:41:55.860023+0000","last_active":"2024-09-21T09:43:06.262868+0000","last_peered":"2024-09-21T09:43:06.262868+0000","last_clean":"2024-09-21T09:43:06.262868+0000","last_became_active":"2024-09-21T09:41:53.843742+0000","last_became_peered":"2024-09-21T09:41:53.843742+0000","last_unstale":"2024-09-21T09:43:06.262868+0000","last_undegraded":"2024-09-21T09:43:06.262868+0000","last_fullsized":"2024-09-21T09:43:06.262868+0000","mapping_epoch":15,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":16,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:14:31.505673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010430319999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"14'97","reported_seq":151,"reported_epoch":21,"state":"active+clean","last_fresh":"2024-09-21T09:43:06.320264+0000","last_change":"2024-09-21T09:41:51.820265+0000","last_active":"2024-09-21T09:43:06.320264+0000","last_peered":"2024-09-21T09:43:06.320264+0000","last_clean":"2024-09-21T09:43:06.320264+0000","last_became_active":"2024-09-21T09:41:51.819950+0000","last_became_peered":"2024-09-21T09:41:51.819950+0000","last_unstale":"2024-09-21T09:43:06.320264+0000","last_undegraded":"2024-09-21T09:43:06.320264+0000","last_fullsized":"2024-09-21T09:43:06.320264+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:50.808452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:50.808452+0000","last_clean_scrub_stamp":"2024-09-21T09:41:50.808452+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:27:36.916853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":9,"seq":38654705684,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27208,"kb_used_data":180,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344632,"statfs":{"total":96636764160,"available":96608903168,"internally_reserved":0,"allocated":184320,"data_stored":52329,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":2,"up_from":11,"seq":47244640277,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27204,"kb_used_data":176,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344636,"statfs":{"total":96636764160,"available":96608907264,"internally_reserved":0,"allocated":180224,"data_stored":52310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":11,"seq":47244640275,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27784,"kb_used_data":752,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344056,"statfs":{"total":96636764160,"available":96608313344,"internally_reserved":0,"allocated":770048,"data_stored":639255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1595,"internal_metadata":27654597},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":11,"seq":47244640277,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27780,"kb_used_data":748,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344060,"statfs":{"total":96636764160,"available":96608317440,"internally_reserved":0,"allocated":765952,"data_stored":639236,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:43:09.651 INFO:tasks.ceph.ceph_manager.ceph:clean! 2024-09-21T09:43:09.651 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T09:43:09.651 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy 2024-09-21T09:43:09.651 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph health --format=json 2024-09-21T09:43:09.749 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.747+0000 7f0ac150f640 1 -- 172.21.15.92:0/2038166064 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0abc070ca0 msgr2=0x7f0abc1333f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:09.750 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.747+0000 7f0ac150f640 1 --2- 172.21.15.92:0/2038166064 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0abc070ca0 0x7f0abc1333f0 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7f0ab4009830 tx=0x7f0ab402a3f0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.750 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.747+0000 7f0ac150f640 1 -- 172.21.15.92:0/2038166064 shutdown_connections 2024-09-21T09:43:09.750 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.747+0000 7f0ac150f640 1 --2- 172.21.15.92:0/2038166064 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0abc070ca0 0x7f0abc1333f0 unknown :-1 s=CLOSED pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.750 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.747+0000 7f0ac150f640 1 --2- 172.21.15.92:0/2038166064 >> v2:172.21.15.92:3300/0 conn(0x7f0abc071510 0x7f0abc130ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.750 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.747+0000 7f0ac150f640 1 -- 172.21.15.92:0/2038166064 >> 172.21.15.92:0/2038166064 conn(0x7f0abc09dcf0 msgr2=0x7f0abc09e100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:09.750 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.747+0000 7f0ac150f640 1 -- 172.21.15.92:0/2038166064 shutdown_connections 2024-09-21T09:43:09.750 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.747+0000 7f0ac150f640 1 -- 172.21.15.92:0/2038166064 wait complete. 2024-09-21T09:43:09.752 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.748+0000 7f0ac150f640 1 Processor -- start 2024-09-21T09:43:09.752 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.748+0000 7f0ac150f640 1 -- start start 2024-09-21T09:43:09.752 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0ac150f640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0abc0701b0 0x7f0abc131470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:09.752 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0ac150f640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0abc070ca0 0x7f0abc1319b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:09.752 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0ac150f640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0abc071510 0x7f0abc131ef0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:09.752 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0ac150f640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f0abc134450 con 0x7f0abc0701b0 2024-09-21T09:43:09.752 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0ac150f640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f0abc1342d0 con 0x7f0abc071510 2024-09-21T09:43:09.752 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0ac150f640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f0abc1345d0 con 0x7f0abc070ca0 2024-09-21T09:43:09.752 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0abaffd640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0abc0701b0 0x7f0abc131470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:09.753 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0aba7fc640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0abc070ca0 0x7f0abc1319b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:09.753 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0aba7fc640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0abc070ca0 0x7f0abc1319b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:49400/0 (socket says 172.21.15.92:49400) 2024-09-21T09:43:09.753 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0abaffd640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0abc0701b0 0x7f0abc131470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:53650/0 (socket says 172.21.15.92:53650) 2024-09-21T09:43:09.753 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0aba7fc640 1 -- 172.21.15.92:0/231737254 learned_addr learned my addr 172.21.15.92:0/231737254 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:09.753 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0abb7fe640 1 --2- 172.21.15.92:0/231737254 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0abc071510 0x7f0abc131ef0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:09.753 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0abaffd640 1 -- 172.21.15.92:0/231737254 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0abc070ca0 msgr2=0x7f0abc1319b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:09.753 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0abaffd640 1 --2- 172.21.15.92:0/231737254 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0abc070ca0 0x7f0abc1319b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.753 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0abaffd640 1 -- 172.21.15.92:0/231737254 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0abc071510 msgr2=0x7f0abc131ef0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:09.753 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0abaffd640 1 --2- 172.21.15.92:0/231737254 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0abc071510 0x7f0abc131ef0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.753 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0abaffd640 1 -- 172.21.15.92:0/231737254 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0abc132ff0 con 0x7f0abc0701b0 2024-09-21T09:43:09.753 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0aba7fc640 1 --2- 172.21.15.92:0/231737254 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0abc070ca0 0x7f0abc1319b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:09.753 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0abb7fe640 1 --2- 172.21.15.92:0/231737254 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0abc071510 0x7f0abc131ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:09.753 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0abaffd640 1 --2- 172.21.15.92:0/231737254 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0abc0701b0 0x7f0abc131470 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f0aac002230 tx=0x7f0aac00cb90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:09.753 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0aa37fe640 1 -- 172.21.15.92:0/231737254 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0aac003b70 con 0x7f0abc0701b0 2024-09-21T09:43:09.754 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0aa37fe640 1 -- 172.21.15.92:0/231737254 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f0aac003d30 con 0x7f0abc0701b0 2024-09-21T09:43:09.754 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0ac150f640 1 -- 172.21.15.92:0/231737254 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0abc0737e0 con 0x7f0abc0701b0 2024-09-21T09:43:09.754 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0aa37fe640 1 -- 172.21.15.92:0/231737254 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0aac0052d0 con 0x7f0abc0701b0 2024-09-21T09:43:09.754 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.749+0000 7f0ac150f640 1 -- 172.21.15.92:0/231737254 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0abc132430 con 0x7f0abc0701b0 2024-09-21T09:43:09.754 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.750+0000 7f0ac150f640 1 -- 172.21.15.92:0/231737254 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0abc070590 con 0x7f0abc0701b0 2024-09-21T09:43:09.754 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.751+0000 7f0aa37fe640 1 -- 172.21.15.92:0/231737254 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 8) ==== 63442+0+0 (secure 0 0 0) 0x7f0aac005680 con 0x7f0abc0701b0 2024-09-21T09:43:09.754 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.751+0000 7f0aa37fe640 1 -- 172.21.15.92:0/231737254 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7f0aac068e30 con 0x7f0abc0701b0 2024-09-21T09:43:09.756 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.754+0000 7f0aa37fe640 1 -- 172.21.15.92:0/231737254 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f0aac067a90 con 0x7f0abc0701b0 2024-09-21T09:43:09.928 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.926+0000 7f0ac150f640 1 -- 172.21.15.92:0/231737254 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "health", "format": "json"} v 0) -- 0x7f0abc096a30 con 0x7f0abc0701b0 2024-09-21T09:43:09.928 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.926+0000 7f0aa37fe640 1 -- 172.21.15.92:0/231737254 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "health", "format": "json"}]=0 v0) ==== 72+0+46 (secure 0 0 0) 0x7f0aac041a60 con 0x7f0abc0701b0 2024-09-21T09:43:09.928 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:09.929 INFO:teuthology.orchestra.run.smithi092.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T09:43:09.930 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.928+0000 7f0ac150f640 1 -- 172.21.15.92:0/231737254 >> v1:172.21.15.92:6800/2299163855 conn(0x7f0aa8051420 legacy=0x7f0aa80538e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:09.930 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.928+0000 7f0ac150f640 1 -- 172.21.15.92:0/231737254 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0abc0701b0 msgr2=0x7f0abc131470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:09.931 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.928+0000 7f0ac150f640 1 --2- 172.21.15.92:0/231737254 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0abc0701b0 0x7f0abc131470 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f0aac002230 tx=0x7f0aac00cb90 comp rx=0 tx=0).stop 2024-09-21T09:43:09.931 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.928+0000 7f0ac150f640 1 -- 172.21.15.92:0/231737254 shutdown_connections 2024-09-21T09:43:09.931 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.928+0000 7f0ac150f640 1 --2- 172.21.15.92:0/231737254 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0abc071510 0x7f0abc131ef0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.931 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.928+0000 7f0ac150f640 1 --2- 172.21.15.92:0/231737254 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0abc070ca0 0x7f0abc1319b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.931 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.928+0000 7f0ac150f640 1 --2- 172.21.15.92:0/231737254 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0abc0701b0 0x7f0abc131470 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:09.931 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.928+0000 7f0ac150f640 1 -- 172.21.15.92:0/231737254 >> 172.21.15.92:0/231737254 conn(0x7f0abc09dcf0 msgr2=0x7f0abc097770 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:09.931 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.928+0000 7f0ac150f640 1 -- 172.21.15.92:0/231737254 shutdown_connections 2024-09-21T09:43:09.931 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:09.928+0000 7f0ac150f640 1 -- 172.21.15.92:0/231737254 wait complete. 2024-09-21T09:43:09.941 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy done 2024-09-21T09:43:09.941 INFO:teuthology.run_tasks:Running task ceph.restart... 2024-09-21T09:43:09.952 DEBUG:tasks.ceph.osd.0:waiting for process to exit 2024-09-21T09:43:09.952 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:43:09.991 INFO:tasks.ceph.osd.0:Stopped 2024-09-21T09:43:09.991 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd down 0 2024-09-21T09:43:10.134 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.128+0000 7f2e025f6640 1 -- 172.21.15.92:0/2782845015 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2dfc1309a0 msgr2=0x7f2dfc132d90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:10.134 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.128+0000 7f2e025f6640 1 --2- 172.21.15.92:0/2782845015 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2dfc1309a0 0x7f2dfc132d90 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7f2df4005fd0 tx=0x7f2df402bb30 comp rx=0 tx=0).stop 2024-09-21T09:43:10.134 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.128+0000 7f2e025f6640 1 -- 172.21.15.92:0/2782845015 shutdown_connections 2024-09-21T09:43:10.134 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.128+0000 7f2e025f6640 1 --2- 172.21.15.92:0/2782845015 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2dfc1309a0 0x7f2dfc132d90 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:10.134 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.128+0000 7f2e025f6640 1 --2- 172.21.15.92:0/2782845015 >> v2:172.21.15.92:3300/0 conn(0x7f2dfc095e70 0x7f2dfc130460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:10.135 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.128+0000 7f2e025f6640 1 -- 172.21.15.92:0/2782845015 >> 172.21.15.92:0/2782845015 conn(0x7f2dfc09afd0 msgr2=0x7f2dfc09b3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:10.135 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.128+0000 7f2e025f6640 1 -- 172.21.15.92:0/2782845015 shutdown_connections 2024-09-21T09:43:10.135 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.128+0000 7f2e025f6640 1 -- 172.21.15.92:0/2782845015 wait complete. 2024-09-21T09:43:10.135 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.129+0000 7f2e025f6640 1 Processor -- start 2024-09-21T09:43:10.135 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.129+0000 7f2e025f6640 1 -- start start 2024-09-21T09:43:10.135 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.129+0000 7f2e025f6640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2dfc095e70 0x7f2dfc08d9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:10.136 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.129+0000 7f2e025f6640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2dfc12b490 0x7f2dfc08def0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:10.136 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.129+0000 7f2e025f6640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2dfc1309a0 0x7f2dfc08e430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:10.136 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.129+0000 7f2e025f6640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f2dfc1334e0 con 0x7f2dfc095e70 2024-09-21T09:43:10.136 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.129+0000 7f2e025f6640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f2dfc094cb0 con 0x7f2dfc12b490 2024-09-21T09:43:10.136 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.129+0000 7f2e025f6640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f2dfc133660 con 0x7f2dfc1309a0 2024-09-21T09:43:10.136 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.130+0000 7f2dfbfff640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2dfc095e70 0x7f2dfc08d9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:10.136 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.130+0000 7f2dfb7fe640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2dfc12b490 0x7f2dfc08def0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:10.137 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.130+0000 7f2e00b93640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2dfc1309a0 0x7f2dfc08e430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:10.137 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.130+0000 7f2dfb7fe640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2dfc12b490 0x7f2dfc08def0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:59626/0 (socket says 172.21.15.92:59626) 2024-09-21T09:43:10.137 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.130+0000 7f2dfbfff640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2dfc095e70 0x7f2dfc08d9b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:53656/0 (socket says 172.21.15.92:53656) 2024-09-21T09:43:10.137 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.130+0000 7f2dfb7fe640 1 -- 172.21.15.92:0/1383591813 learned_addr learned my addr 172.21.15.92:0/1383591813 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:10.137 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.130+0000 7f2e00b93640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2dfc1309a0 0x7f2dfc08e430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:49412/0 (socket says 172.21.15.92:49412) 2024-09-21T09:43:10.137 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.130+0000 7f2dfbfff640 1 -- 172.21.15.92:0/1383591813 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2dfc1309a0 msgr2=0x7f2dfc08e430 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:10.137 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.130+0000 7f2dfbfff640 1 --2- 172.21.15.92:0/1383591813 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2dfc1309a0 0x7f2dfc08e430 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:10.137 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.130+0000 7f2dfbfff640 1 -- 172.21.15.92:0/1383591813 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2dfc12b490 msgr2=0x7f2dfc08def0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:10.137 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.130+0000 7f2dfbfff640 1 --2- 172.21.15.92:0/1383591813 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2dfc12b490 0x7f2dfc08def0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:10.137 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.130+0000 7f2dfbfff640 1 -- 172.21.15.92:0/1383591813 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2dfc08ff30 con 0x7f2dfc095e70 2024-09-21T09:43:10.137 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.130+0000 7f2dfb7fe640 1 --2- 172.21.15.92:0/1383591813 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2dfc12b490 0x7f2dfc08def0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:10.137 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.130+0000 7f2e00b93640 1 --2- 172.21.15.92:0/1383591813 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2dfc1309a0 0x7f2dfc08e430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:10.137 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.130+0000 7f2dfbfff640 1 --2- 172.21.15.92:0/1383591813 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2dfc095e70 0x7f2dfc08d9b0 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f2dec00c3c0 tx=0x7f2dec00c880 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:10.138 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.130+0000 7f2df8ff9640 1 -- 172.21.15.92:0/1383591813 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2dec011790 con 0x7f2dfc095e70 2024-09-21T09:43:10.138 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.130+0000 7f2df8ff9640 1 -- 172.21.15.92:0/1383591813 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2dec017070 con 0x7f2dfc095e70 2024-09-21T09:43:10.138 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.130+0000 7f2e025f6640 1 -- 172.21.15.92:0/1383591813 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2dfc08e970 con 0x7f2dfc095e70 2024-09-21T09:43:10.138 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.130+0000 7f2df8ff9640 1 -- 172.21.15.92:0/1383591813 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2dec011d20 con 0x7f2dfc095e70 2024-09-21T09:43:10.138 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.130+0000 7f2e025f6640 1 -- 172.21.15.92:0/1383591813 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2dfc090580 con 0x7f2dfc095e70 2024-09-21T09:43:10.138 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.131+0000 7f2df8ff9640 1 -- 172.21.15.92:0/1383591813 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 8) ==== 63442+0+0 (secure 0 0 0) 0x7f2dec01c460 con 0x7f2dfc095e70 2024-09-21T09:43:10.138 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.131+0000 7f2e025f6640 1 -- 172.21.15.92:0/1383591813 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2dfc12b870 con 0x7f2dfc095e70 2024-09-21T09:43:10.138 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.132+0000 7f2df8ff9640 1 -- 172.21.15.92:0/1383591813 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7f2dec068cc0 con 0x7f2dfc095e70 2024-09-21T09:43:10.138 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.135+0000 7f2df8ff9640 1 -- 172.21.15.92:0/1383591813 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f2dec006890 con 0x7f2dfc095e70 2024-09-21T09:43:10.271 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.269+0000 7f2df8ff9640 1 -- 172.21.15.92:0/1383591813 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f2dec0675a0 con 0x7f2dfc095e70 2024-09-21T09:43:10.298 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.296+0000 7f2e025f6640 1 -- 172.21.15.92:0/1383591813 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd down", "ids": ["0"]} v 0) -- 0x7f2dfc12db50 con 0x7f2dfc095e70 2024-09-21T09:43:10.299 INFO:tasks.ceph.mon.a.smithi092.stderr:2024-09-21T09:43:10.297+0000 7fe01bdd1640 -1 mon.a@0(leader).osd e22 definitely_dead 0 2024-09-21T09:43:10.299 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.297+0000 7f2df8ff9640 1 -- 172.21.15.92:0/1383591813 <== mon.0 v2:172.21.15.92:3300/0 8 ==== mon_command_ack([{"prefix": "osd down", "ids": ["0"]}]=0 osd.0 is already down. v22) ==== 93+0+0 (secure 0 0 0) 0x7f2dec011cc0 con 0x7f2dfc095e70 2024-09-21T09:43:10.300 INFO:teuthology.orchestra.run.smithi092.stderr:osd.0 is already down. 2024-09-21T09:43:10.301 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.299+0000 7f2e025f6640 1 -- 172.21.15.92:0/1383591813 >> v1:172.21.15.92:6800/2299163855 conn(0x7f2de8051420 legacy=0x7f2de80538e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:10.301 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.299+0000 7f2e025f6640 1 -- 172.21.15.92:0/1383591813 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2dfc095e70 msgr2=0x7f2dfc08d9b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:10.301 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.299+0000 7f2e025f6640 1 --2- 172.21.15.92:0/1383591813 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2dfc095e70 0x7f2dfc08d9b0 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f2dec00c3c0 tx=0x7f2dec00c880 comp rx=0 tx=0).stop 2024-09-21T09:43:10.302 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.299+0000 7f2e025f6640 1 -- 172.21.15.92:0/1383591813 shutdown_connections 2024-09-21T09:43:10.302 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.299+0000 7f2e025f6640 1 --2- 172.21.15.92:0/1383591813 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2dfc1309a0 0x7f2dfc08e430 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:10.302 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.299+0000 7f2e025f6640 1 --2- 172.21.15.92:0/1383591813 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2dfc12b490 0x7f2dfc08def0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:10.302 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.299+0000 7f2e025f6640 1 --2- 172.21.15.92:0/1383591813 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2dfc095e70 0x7f2dfc08d9b0 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:10.302 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.299+0000 7f2e025f6640 1 -- 172.21.15.92:0/1383591813 >> 172.21.15.92:0/1383591813 conn(0x7f2dfc09afd0 msgr2=0x7f2dfc092ef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:10.302 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.299+0000 7f2e025f6640 1 -- 172.21.15.92:0/1383591813 shutdown_connections 2024-09-21T09:43:10.303 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.299+0000 7f2e025f6640 1 -- 172.21.15.92:0/1383591813 wait complete. 2024-09-21T09:43:10.313 INFO:tasks.ceph.osd.0:Restarting daemon 2024-09-21T09:43:10.314 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 0 2024-09-21T09:43:10.317 INFO:tasks.ceph.osd.0:Started 2024-09-21T09:43:10.317 DEBUG:tasks.ceph.osd.1:waiting for process to exit 2024-09-21T09:43:10.317 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:43:10.362 INFO:tasks.ceph.osd.1:Stopped 2024-09-21T09:43:10.362 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd down 1 2024-09-21T09:43:10.464 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.462+0000 7f2629a27640 1 -- 172.21.15.92:0/3407610965 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f26240953a0 msgr2=0x7f2624095830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:10.464 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.462+0000 7f2629a27640 1 --2- 172.21.15.92:0/3407610965 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f26240953a0 0x7f2624095830 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f261c00acf0 tx=0x7f261c02bac0 comp rx=0 tx=0).stop 2024-09-21T09:43:10.465 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.462+0000 7f2629a27640 1 -- 172.21.15.92:0/3407610965 shutdown_connections 2024-09-21T09:43:10.465 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.462+0000 7f2629a27640 1 --2- 172.21.15.92:0/3407610965 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f26240953a0 0x7f2624095830 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:10.465 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.462+0000 7f2629a27640 1 --2- 172.21.15.92:0/3407610965 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f262412bdb0 0x7f2624094e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:10.465 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.462+0000 7f2629a27640 1 --2- 172.21.15.92:0/3407610965 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f262412b490 0x7f262412b870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:10.465 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.462+0000 7f2629a27640 1 -- 172.21.15.92:0/3407610965 >> 172.21.15.92:0/3407610965 conn(0x7f262409afd0 msgr2=0x7f262409b3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:10.465 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.462+0000 7f2629a27640 1 -- 172.21.15.92:0/3407610965 shutdown_connections 2024-09-21T09:43:10.465 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.462+0000 7f2629a27640 1 -- 172.21.15.92:0/3407610965 wait complete. 2024-09-21T09:43:10.467 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.463+0000 7f2629a27640 1 Processor -- start 2024-09-21T09:43:10.467 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.463+0000 7f2629a27640 1 -- start start 2024-09-21T09:43:10.467 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f2629a27640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f26240953a0 0x7f26241c4a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:10.467 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f2629a27640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f262412b490 0x7f26241c4fa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:10.468 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f2629a27640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f262412bdb0 0x7f2624072a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:10.468 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f2629a27640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f2624132470 con 0x7f262412b490 2024-09-21T09:43:10.468 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f2629a27640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f2624094b90 con 0x7f26240953a0 2024-09-21T09:43:10.468 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f2629a27640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f26241325f0 con 0x7f262412bdb0 2024-09-21T09:43:10.468 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f2622ffd640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f26240953a0 0x7f26241c4a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:10.468 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f26227fc640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f262412b490 0x7f26241c4fa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:10.468 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f26237fe640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f262412bdb0 0x7f2624072a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:10.468 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f2622ffd640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f26240953a0 0x7f26241c4a60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:59646/0 (socket says 172.21.15.92:59646) 2024-09-21T09:43:10.468 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f2622ffd640 1 -- 172.21.15.92:0/2725361618 learned_addr learned my addr 172.21.15.92:0/2725361618 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:10.468 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f2622ffd640 1 -- 172.21.15.92:0/2725361618 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f262412bdb0 msgr2=0x7f2624072a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:10.468 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f2622ffd640 1 --2- 172.21.15.92:0/2725361618 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f262412bdb0 0x7f2624072a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:10.468 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f2622ffd640 1 -- 172.21.15.92:0/2725361618 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f262412b490 msgr2=0x7f26241c4fa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:10.468 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f2622ffd640 1 --2- 172.21.15.92:0/2725361618 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f262412b490 0x7f26241c4fa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:10.468 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f2622ffd640 1 -- 172.21.15.92:0/2725361618 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f262408d7d0 con 0x7f26240953a0 2024-09-21T09:43:10.469 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f26237fe640 1 --2- 172.21.15.92:0/2725361618 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f262412bdb0 0x7f2624072a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:10.469 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f26227fc640 1 --2- 172.21.15.92:0/2725361618 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f262412b490 0x7f26241c4fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:10.469 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f2622ffd640 1 --2- 172.21.15.92:0/2725361618 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f26240953a0 0x7f26241c4a60 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f261400ea90 tx=0x7f261400ef50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:10.469 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f260bfff640 1 -- 172.21.15.92:0/2725361618 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f261400ce40 con 0x7f26240953a0 2024-09-21T09:43:10.469 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f260bfff640 1 -- 172.21.15.92:0/2725361618 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2614004020 con 0x7f26240953a0 2024-09-21T09:43:10.469 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f260bfff640 1 -- 172.21.15.92:0/2725361618 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2614004470 con 0x7f26240953a0 2024-09-21T09:43:10.469 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.464+0000 7f2629a27640 1 -- 172.21.15.92:0/2725361618 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f262408c4f0 con 0x7f26240953a0 2024-09-21T09:43:10.469 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.465+0000 7f2629a27640 1 -- 172.21.15.92:0/2725361618 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f26241c54e0 con 0x7f26240953a0 2024-09-21T09:43:10.469 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.465+0000 7f2629a27640 1 -- 172.21.15.92:0/2725361618 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2624095830 con 0x7f26240953a0 2024-09-21T09:43:10.469 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.466+0000 7f260bfff640 1 -- 172.21.15.92:0/2725361618 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f26140035f0 con 0x7f26240953a0 2024-09-21T09:43:10.469 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.466+0000 7f260bfff640 1 -- 172.21.15.92:0/2725361618 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(22..22 src has 1..22) ==== 3027+0+0 (secure 0 0 0) 0x7f2614069a10 con 0x7f26240953a0 2024-09-21T09:43:10.471 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.470+0000 7f260bfff640 1 -- 172.21.15.92:0/2725361618 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f2614068ac0 con 0x7f26240953a0 2024-09-21T09:43:10.628 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:10.626+0000 7f2629a27640 1 -- 172.21.15.92:0/2725361618 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "osd down", "ids": ["1"]} v 0) -- 0x7f26240699c0 con 0x7f26240953a0 2024-09-21T09:43:10.628 INFO:tasks.ceph.mon.a.smithi092.stderr:2024-09-21T09:43:10.627+0000 7fe01bdd1640 -1 mon.a@0(leader).osd e22 definitely_dead 0 2024-09-21T09:43:11.277 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.275+0000 7f260bfff640 1 -- 172.21.15.92:0/2725361618 <== mon.1 v2:172.21.15.92:3301/0 7 ==== mon_command_ack([{"prefix": "osd down", "ids": ["1"]}]=0 marked down osd.1. v23) ==== 89+0+0 (secure 0 0 0) 0x7f2614068600 con 0x7f26240953a0 2024-09-21T09:43:11.277 INFO:teuthology.orchestra.run.smithi092.stderr:marked down osd.1. 2024-09-21T09:43:11.280 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.277+0000 7f2629a27640 1 -- 172.21.15.92:0/2725361618 >> v1:172.21.15.92:6800/2299163855 conn(0x7f26100514b0 legacy=0x7f2610053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:11.280 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.278+0000 7f2629a27640 1 -- 172.21.15.92:0/2725361618 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f26240953a0 msgr2=0x7f26241c4a60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:11.280 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.278+0000 7f2629a27640 1 --2- 172.21.15.92:0/2725361618 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f26240953a0 0x7f26241c4a60 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f261400ea90 tx=0x7f261400ef50 comp rx=0 tx=0).stop 2024-09-21T09:43:11.280 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.278+0000 7f2629a27640 1 -- 172.21.15.92:0/2725361618 shutdown_connections 2024-09-21T09:43:11.280 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.278+0000 7f2629a27640 1 --2- 172.21.15.92:0/2725361618 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f262412bdb0 0x7f2624072a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:11.280 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.278+0000 7f2629a27640 1 --2- 172.21.15.92:0/2725361618 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f262412b490 0x7f26241c4fa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:11.280 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.278+0000 7f2629a27640 1 --2- 172.21.15.92:0/2725361618 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f26240953a0 0x7f26241c4a60 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:11.280 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.278+0000 7f2629a27640 1 -- 172.21.15.92:0/2725361618 >> 172.21.15.92:0/2725361618 conn(0x7f262409afd0 msgr2=0x7f2624092560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:11.280 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.278+0000 7f2629a27640 1 -- 172.21.15.92:0/2725361618 shutdown_connections 2024-09-21T09:43:11.280 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.278+0000 7f2629a27640 1 -- 172.21.15.92:0/2725361618 wait complete. 2024-09-21T09:43:11.291 INFO:tasks.ceph.osd.1:Restarting daemon 2024-09-21T09:43:11.292 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 1 2024-09-21T09:43:11.294 INFO:tasks.ceph.osd.1:Started 2024-09-21T09:43:11.294 DEBUG:tasks.ceph.osd.2:waiting for process to exit 2024-09-21T09:43:11.294 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:43:11.323 INFO:tasks.ceph.osd.2:Stopped 2024-09-21T09:43:11.323 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd down 2 2024-09-21T09:43:11.463 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.461+0000 7f368fcbd640 1 -- 172.21.15.92:0/4142197329 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f368812b490 msgr2=0x7f368812b870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:11.463 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.461+0000 7f368fcbd640 1 --2- 172.21.15.92:0/4142197329 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f368812b490 0x7f368812b870 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f367c005fd0 tx=0x7f367c02a540 comp rx=0 tx=0).stop 2024-09-21T09:43:11.463 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.461+0000 7f368fcbd640 1 -- 172.21.15.92:0/4142197329 shutdown_connections 2024-09-21T09:43:11.463 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.461+0000 7f368fcbd640 1 --2- 172.21.15.92:0/4142197329 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f3688095570 0x7f36880959e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:11.463 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.461+0000 7f368fcbd640 1 --2- 172.21.15.92:0/4142197329 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f368812b490 0x7f368812b870 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:11.464 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.461+0000 7f368fcbd640 1 -- 172.21.15.92:0/4142197329 >> 172.21.15.92:0/4142197329 conn(0x7f368809afd0 msgr2=0x7f368809b3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:11.464 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.461+0000 7f368fcbd640 1 -- 172.21.15.92:0/4142197329 shutdown_connections 2024-09-21T09:43:11.464 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.461+0000 7f368fcbd640 1 -- 172.21.15.92:0/4142197329 wait complete. 2024-09-21T09:43:11.466 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.462+0000 7f368fcbd640 1 Processor -- start 2024-09-21T09:43:11.466 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.462+0000 7f368fcbd640 1 -- start start 2024-09-21T09:43:11.466 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.462+0000 7f368fcbd640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f3688095570 0x7f3688072ad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:11.466 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.462+0000 7f368fcbd640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f368812b490 0x7f3688073010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:11.466 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.462+0000 7f368fcbd640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f368812bdb0 0x7f3688073550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:11.466 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.462+0000 7f368fcbd640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f3688133dc0 con 0x7f3688095570 2024-09-21T09:43:11.466 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.462+0000 7f368fcbd640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f3688133c40 con 0x7f368812bdb0 2024-09-21T09:43:11.466 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.462+0000 7f368fcbd640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f3688133f40 con 0x7f368812b490 2024-09-21T09:43:11.466 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.462+0000 7f368da59640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f3688095570 0x7f3688072ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:11.466 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.462+0000 7f368da59640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f3688095570 0x7f3688072ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:53718/0 (socket says 172.21.15.92:53718) 2024-09-21T09:43:11.467 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.462+0000 7f368da59640 1 -- 172.21.15.92:0/134845895 learned_addr learned my addr 172.21.15.92:0/134845895 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:11.467 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.462+0000 7f368d258640 1 --2- 172.21.15.92:0/134845895 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f368812b490 0x7f3688073010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:11.467 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.463+0000 7f368e25a640 1 --2- 172.21.15.92:0/134845895 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f368812bdb0 0x7f3688073550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:11.467 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.463+0000 7f368da59640 1 -- 172.21.15.92:0/134845895 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f368812b490 msgr2=0x7f3688073010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:11.467 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.463+0000 7f368da59640 1 --2- 172.21.15.92:0/134845895 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f368812b490 0x7f3688073010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:11.467 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.463+0000 7f368da59640 1 -- 172.21.15.92:0/134845895 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f368812bdb0 msgr2=0x7f3688073550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:11.467 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.463+0000 7f368da59640 1 --2- 172.21.15.92:0/134845895 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f368812bdb0 0x7f3688073550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:11.467 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.463+0000 7f368da59640 1 -- 172.21.15.92:0/134845895 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3688074380 con 0x7f3688095570 2024-09-21T09:43:11.467 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.463+0000 7f368da59640 1 --2- 172.21.15.92:0/134845895 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f3688095570 0x7f3688072ad0 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f367c009850 tx=0x7f367c0038a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:11.467 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.463+0000 7f367a7fc640 1 -- 172.21.15.92:0/134845895 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f367c03f070 con 0x7f3688095570 2024-09-21T09:43:11.467 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.463+0000 7f367a7fc640 1 -- 172.21.15.92:0/134845895 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f367c0047a0 con 0x7f3688095570 2024-09-21T09:43:11.467 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.463+0000 7f368fcbd640 1 -- 172.21.15.92:0/134845895 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f368812d4f0 con 0x7f3688095570 2024-09-21T09:43:11.468 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.463+0000 7f367a7fc640 1 -- 172.21.15.92:0/134845895 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f367c035030 con 0x7f3688095570 2024-09-21T09:43:11.468 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.463+0000 7f368fcbd640 1 -- 172.21.15.92:0/134845895 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f36881c25a0 con 0x7f3688095570 2024-09-21T09:43:11.468 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.464+0000 7f368fcbd640 1 -- 172.21.15.92:0/134845895 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f36880959e0 con 0x7f3688095570 2024-09-21T09:43:11.468 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.464+0000 7f367a7fc640 1 -- 172.21.15.92:0/134845895 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f367c042070 con 0x7f3688095570 2024-09-21T09:43:11.468 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.465+0000 7f367a7fc640 1 -- 172.21.15.92:0/134845895 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(23..23 src has 1..23) ==== 3027+0+0 (secure 0 0 0) 0x7f367c08a020 con 0x7f3688095570 2024-09-21T09:43:11.470 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.468+0000 7f367a7fc640 1 -- 172.21.15.92:0/134845895 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f367c0890d0 con 0x7f3688095570 2024-09-21T09:43:11.630 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:11.628+0000 7f368fcbd640 1 -- 172.21.15.92:0/134845895 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd down", "ids": ["2"]} v 0) -- 0x7f36880699c0 con 0x7f3688095570 2024-09-21T09:43:11.630 INFO:tasks.ceph.mon.a.smithi092.stderr:2024-09-21T09:43:11.628+0000 7fe01bdd1640 -1 mon.a@0(leader).osd e23 definitely_dead 0 2024-09-21T09:43:11.737 INFO:tasks.ceph.osd.0.smithi092.stderr:2024-09-21T09:43:11.735+0000 7f3a7140b640 -1 Falling back to public interface 2024-09-21T09:43:12.278 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.276+0000 7f367a7fc640 1 -- 172.21.15.92:0/134845895 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd down", "ids": ["2"]}]=0 marked down osd.2. v24) ==== 89+0+0 (secure 0 0 0) 0x7f367c088c10 con 0x7f3688095570 2024-09-21T09:43:12.278 INFO:teuthology.orchestra.run.smithi092.stderr:marked down osd.2. 2024-09-21T09:43:12.280 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.279+0000 7f368fcbd640 1 -- 172.21.15.92:0/134845895 >> v1:172.21.15.92:6800/2299163855 conn(0x7f366c0514b0 legacy=0x7f366c053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:12.281 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.279+0000 7f368fcbd640 1 -- 172.21.15.92:0/134845895 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f3688095570 msgr2=0x7f3688072ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:12.281 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.279+0000 7f368fcbd640 1 --2- 172.21.15.92:0/134845895 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f3688095570 0x7f3688072ad0 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f367c009850 tx=0x7f367c0038a0 comp rx=0 tx=0).stop 2024-09-21T09:43:12.281 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.279+0000 7f368fcbd640 1 -- 172.21.15.92:0/134845895 shutdown_connections 2024-09-21T09:43:12.281 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.279+0000 7f368fcbd640 1 --2- 172.21.15.92:0/134845895 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f368812bdb0 0x7f3688073550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:12.281 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.279+0000 7f368fcbd640 1 --2- 172.21.15.92:0/134845895 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f368812b490 0x7f3688073010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:12.281 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.279+0000 7f368fcbd640 1 --2- 172.21.15.92:0/134845895 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f3688095570 0x7f3688072ad0 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:12.281 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.279+0000 7f368fcbd640 1 -- 172.21.15.92:0/134845895 >> 172.21.15.92:0/134845895 conn(0x7f368809afd0 msgr2=0x7f36880901a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:12.281 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.279+0000 7f368fcbd640 1 -- 172.21.15.92:0/134845895 shutdown_connections 2024-09-21T09:43:12.281 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.279+0000 7f368fcbd640 1 -- 172.21.15.92:0/134845895 wait complete. 2024-09-21T09:43:12.293 INFO:tasks.ceph.osd.2:Restarting daemon 2024-09-21T09:43:12.293 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 2 2024-09-21T09:43:12.295 INFO:tasks.ceph.osd.2:Started 2024-09-21T09:43:12.296 DEBUG:tasks.ceph.osd.3:waiting for process to exit 2024-09-21T09:43:12.296 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:43:12.330 INFO:tasks.ceph.osd.3:Stopped 2024-09-21T09:43:12.330 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd down 3 2024-09-21T09:43:12.471 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.469+0000 7f4036f66640 1 -- 172.21.15.92:0/3924107265 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4030095120 msgr2=0x7f40301327a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:12.471 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.469+0000 7f4036f66640 1 --2- 172.21.15.92:0/3924107265 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4030095120 0x7f40301327a0 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f4028009ca0 tx=0x7f402802a9a0 comp rx=0 tx=0).stop 2024-09-21T09:43:12.471 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.469+0000 7f4036f66640 1 -- 172.21.15.92:0/3924107265 shutdown_connections 2024-09-21T09:43:12.471 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.469+0000 7f4036f66640 1 --2- 172.21.15.92:0/3924107265 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4030095120 0x7f40301327a0 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:12.472 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.469+0000 7f4036f66640 1 --2- 172.21.15.92:0/3924107265 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4030094ce0 0x7f4030130250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:12.472 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.469+0000 7f4036f66640 1 -- 172.21.15.92:0/3924107265 >> 172.21.15.92:0/3924107265 conn(0x7f403009afd0 msgr2=0x7f403009b3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:12.472 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.469+0000 7f4036f66640 1 -- 172.21.15.92:0/3924107265 shutdown_connections 2024-09-21T09:43:12.472 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.469+0000 7f4036f66640 1 -- 172.21.15.92:0/3924107265 wait complete. 2024-09-21T09:43:12.474 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.470+0000 7f4036f66640 1 Processor -- start 2024-09-21T09:43:12.474 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.470+0000 7f4036f66640 1 -- start start 2024-09-21T09:43:12.474 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f4036f66640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4030093e40 0x7f4030124c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:12.474 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f4036f66640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4030094ce0 0x7f4030125190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:12.474 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f4036f66640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4030095120 0x7f40301256d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:12.474 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f4036f66640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f4030133490 con 0x7f4030093e40 2024-09-21T09:43:12.474 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f4036f66640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f4030133310 con 0x7f4030094ce0 2024-09-21T09:43:12.474 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f4036f66640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f4030133610 con 0x7f4030095120 2024-09-21T09:43:12.475 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f4034d02640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4030093e40 0x7f4030124c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:12.475 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f4027fff640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4030094ce0 0x7f4030125190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:12.475 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f4035503640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4030095120 0x7f40301256d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:12.475 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f4034d02640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4030093e40 0x7f4030124c50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:53750/0 (socket says 172.21.15.92:53750) 2024-09-21T09:43:12.475 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f4034d02640 1 -- 172.21.15.92:0/592220911 learned_addr learned my addr 172.21.15.92:0/592220911 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:12.475 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f4034d02640 1 -- 172.21.15.92:0/592220911 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4030095120 msgr2=0x7f40301256d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:12.475 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f4034d02640 1 --2- 172.21.15.92:0/592220911 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4030095120 0x7f40301256d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:12.475 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f4034d02640 1 -- 172.21.15.92:0/592220911 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4030094ce0 msgr2=0x7f4030125190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:12.475 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f4034d02640 1 --2- 172.21.15.92:0/592220911 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4030094ce0 0x7f4030125190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:12.475 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f4034d02640 1 -- 172.21.15.92:0/592220911 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4030127d90 con 0x7f4030093e40 2024-09-21T09:43:12.475 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f4027fff640 1 --2- 172.21.15.92:0/592220911 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4030094ce0 0x7f4030125190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:12.475 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f4034d02640 1 --2- 172.21.15.92:0/592220911 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4030093e40 0x7f4030124c50 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f402c00c400 tx=0x7f402c00c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:12.475 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f40257fa640 1 -- 172.21.15.92:0/592220911 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f402c017070 con 0x7f4030093e40 2024-09-21T09:43:12.476 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f40257fa640 1 -- 172.21.15.92:0/592220911 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f402c0053c0 con 0x7f4030093e40 2024-09-21T09:43:12.476 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f4036f66640 1 -- 172.21.15.92:0/592220911 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4030126540 con 0x7f4030093e40 2024-09-21T09:43:12.476 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f40257fa640 1 -- 172.21.15.92:0/592220911 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f402c003850 con 0x7f4030093e40 2024-09-21T09:43:12.476 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.471+0000 7f4036f66640 1 -- 172.21.15.92:0/592220911 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f40301c4b00 con 0x7f4030093e40 2024-09-21T09:43:12.476 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.472+0000 7f4036f66640 1 -- 172.21.15.92:0/592220911 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f40300945d0 con 0x7f4030093e40 2024-09-21T09:43:12.476 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.472+0000 7f40257fa640 1 -- 172.21.15.92:0/592220911 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f402c01a070 con 0x7f4030093e40 2024-09-21T09:43:12.476 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.473+0000 7f40257fa640 1 -- 172.21.15.92:0/592220911 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(24..24 src has 1..24) ==== 3027+0+0 (secure 0 0 0) 0x7f402c0683d0 con 0x7f4030093e40 2024-09-21T09:43:12.478 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.476+0000 7f40257fa640 1 -- 172.21.15.92:0/592220911 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f402c067480 con 0x7f4030093e40 2024-09-21T09:43:12.635 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:12.633+0000 7f4036f66640 1 -- 172.21.15.92:0/592220911 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd down", "ids": ["3"]} v 0) -- 0x7f4030127900 con 0x7f4030093e40 2024-09-21T09:43:12.636 INFO:tasks.ceph.mon.a.smithi092.stderr:2024-09-21T09:43:12.634+0000 7fe01bdd1640 -1 mon.a@0(leader).osd e24 definitely_dead 0 2024-09-21T09:43:13.220 INFO:tasks.ceph.osd.1.smithi092.stderr:2024-09-21T09:43:13.218+0000 7f0791c64640 -1 Falling back to public interface 2024-09-21T09:43:13.280 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.278+0000 7f40257fa640 1 -- 172.21.15.92:0/592220911 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd down", "ids": ["3"]}]=0 marked down osd.3. v25) ==== 89+0+0 (secure 0 0 0) 0x7f402c066c40 con 0x7f4030093e40 2024-09-21T09:43:13.280 INFO:teuthology.orchestra.run.smithi092.stderr:marked down osd.3. 2024-09-21T09:43:13.282 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.280+0000 7f4036f66640 1 -- 172.21.15.92:0/592220911 >> v1:172.21.15.92:6800/2299163855 conn(0x7f401c0514b0 legacy=0x7f401c053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:13.282 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.280+0000 7f4036f66640 1 -- 172.21.15.92:0/592220911 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4030093e40 msgr2=0x7f4030124c50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:13.282 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.280+0000 7f4036f66640 1 --2- 172.21.15.92:0/592220911 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4030093e40 0x7f4030124c50 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f402c00c400 tx=0x7f402c00c8c0 comp rx=0 tx=0).stop 2024-09-21T09:43:13.282 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.280+0000 7f4036f66640 1 -- 172.21.15.92:0/592220911 shutdown_connections 2024-09-21T09:43:13.282 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.280+0000 7f4036f66640 1 --2- 172.21.15.92:0/592220911 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4030095120 0x7f40301256d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:13.282 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.280+0000 7f4036f66640 1 --2- 172.21.15.92:0/592220911 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4030094ce0 0x7f4030125190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:13.282 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.280+0000 7f4036f66640 1 --2- 172.21.15.92:0/592220911 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4030093e40 0x7f4030124c50 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:13.282 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.280+0000 7f4036f66640 1 -- 172.21.15.92:0/592220911 >> 172.21.15.92:0/592220911 conn(0x7f403009afd0 msgr2=0x7f4030072b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:13.282 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.280+0000 7f4036f66640 1 -- 172.21.15.92:0/592220911 shutdown_connections 2024-09-21T09:43:13.282 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.280+0000 7f4036f66640 1 -- 172.21.15.92:0/592220911 wait complete. 2024-09-21T09:43:13.295 INFO:tasks.ceph.osd.3:Restarting daemon 2024-09-21T09:43:13.295 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 3 2024-09-21T09:43:13.297 INFO:tasks.ceph.osd.3:Started 2024-09-21T09:43:13.298 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up 2024-09-21T09:43:13.298 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:43:13.399 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.397+0000 7f24a4cb3640 1 -- 172.21.15.92:0/3362418882 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f24a012b490 msgr2=0x7f24a012b870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:13.399 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.397+0000 7f24a4cb3640 1 --2- 172.21.15.92:0/3362418882 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f24a012b490 0x7f24a012b870 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7f2490009830 tx=0x7f249002a360 comp rx=0 tx=0).stop 2024-09-21T09:43:13.399 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.397+0000 7f24a4cb3640 1 -- 172.21.15.92:0/3362418882 shutdown_connections 2024-09-21T09:43:13.399 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.397+0000 7f24a4cb3640 1 --2- 172.21.15.92:0/3362418882 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f24a0094b10 0x7f24a0094f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:13.399 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.397+0000 7f24a4cb3640 1 --2- 172.21.15.92:0/3362418882 >> v2:172.21.15.92:3300/0 conn(0x7f24a012bdb0 0x7f24a00945d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:13.399 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.397+0000 7f24a4cb3640 1 --2- 172.21.15.92:0/3362418882 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f24a012b490 0x7f24a012b870 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:13.400 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.397+0000 7f24a4cb3640 1 -- 172.21.15.92:0/3362418882 >> 172.21.15.92:0/3362418882 conn(0x7f24a009a2e0 msgr2=0x7f24a0097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:13.400 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.397+0000 7f24a4cb3640 1 -- 172.21.15.92:0/3362418882 shutdown_connections 2024-09-21T09:43:13.400 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.397+0000 7f24a4cb3640 1 -- 172.21.15.92:0/3362418882 wait complete. 2024-09-21T09:43:13.402 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.398+0000 7f24a4cb3640 1 Processor -- start 2024-09-21T09:43:13.402 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.398+0000 7f24a4cb3640 1 -- start start 2024-09-21T09:43:13.402 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.398+0000 7f24a4cb3640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f24a0094b10 0x7f24a01304d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:13.402 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.398+0000 7f24a4cb3640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f24a012b490 0x7f24a0130a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:13.403 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.398+0000 7f24a4cb3640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f24a012bdb0 0x7f24a01c2840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:13.403 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.398+0000 7f24a4cb3640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f24a0132470 con 0x7f24a012b490 2024-09-21T09:43:13.403 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.398+0000 7f24a4cb3640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f24a00997e0 con 0x7f24a012bdb0 2024-09-21T09:43:13.403 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.398+0000 7f24a4cb3640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f24a01325f0 con 0x7f24a0094b10 2024-09-21T09:43:13.403 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.399+0000 7f249e59c640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f24a0094b10 0x7f24a01304d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:13.403 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.399+0000 7f249ed9d640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f24a012bdb0 0x7f24a01c2840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:13.403 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.399+0000 7f249dd9b640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f24a012b490 0x7f24a0130a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:13.403 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.399+0000 7f249e59c640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f24a0094b10 0x7f24a01304d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:49514/0 (socket says 172.21.15.92:49514) 2024-09-21T09:43:13.403 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.399+0000 7f249e59c640 1 -- 172.21.15.92:0/4134367717 learned_addr learned my addr 172.21.15.92:0/4134367717 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:13.403 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.399+0000 7f249dd9b640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f24a012b490 0x7f24a0130a10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:53774/0 (socket says 172.21.15.92:53774) 2024-09-21T09:43:13.403 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.399+0000 7f249e59c640 1 -- 172.21.15.92:0/4134367717 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f24a012bdb0 msgr2=0x7f24a01c2840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:13.403 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.399+0000 7f249e59c640 1 --2- 172.21.15.92:0/4134367717 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f24a012bdb0 0x7f24a01c2840 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:13.403 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.399+0000 7f249e59c640 1 -- 172.21.15.92:0/4134367717 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f24a012b490 msgr2=0x7f24a0130a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:13.404 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.399+0000 7f249e59c640 1 --2- 172.21.15.92:0/4134367717 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f24a012b490 0x7f24a0130a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:13.404 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.399+0000 7f249e59c640 1 -- 172.21.15.92:0/4134367717 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f24a0071970 con 0x7f24a0094b10 2024-09-21T09:43:13.404 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.399+0000 7f249dd9b640 1 --2- 172.21.15.92:0/4134367717 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f24a012b490 0x7f24a0130a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:13.404 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.399+0000 7f249ed9d640 1 --2- 172.21.15.92:0/4134367717 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f24a012bdb0 0x7f24a01c2840 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:13.404 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.399+0000 7f249e59c640 1 --2- 172.21.15.92:0/4134367717 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f24a0094b10 0x7f24a01304d0 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f2490009780 tx=0x7f2490033b70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:13.404 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.399+0000 7f248effd640 1 -- 172.21.15.92:0/4134367717 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f24900044c0 con 0x7f24a0094b10 2024-09-21T09:43:13.404 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.399+0000 7f248effd640 1 -- 172.21.15.92:0/4134367717 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f24900035f0 con 0x7f24a0094b10 2024-09-21T09:43:13.404 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.399+0000 7f24a4cb3640 1 -- 172.21.15.92:0/4134367717 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f24a0131b90 con 0x7f24a0094b10 2024-09-21T09:43:13.404 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.399+0000 7f248effd640 1 -- 172.21.15.92:0/4134367717 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2490003a40 con 0x7f24a0094b10 2024-09-21T09:43:13.404 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.399+0000 7f24a4cb3640 1 -- 172.21.15.92:0/4134367717 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f24a0130f50 con 0x7f24a0094b10 2024-09-21T09:43:13.404 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.400+0000 7f24a4cb3640 1 -- 172.21.15.92:0/4134367717 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f24a0092eb0 con 0x7f24a0094b10 2024-09-21T09:43:13.404 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.400+0000 7f248effd640 1 -- 172.21.15.92:0/4134367717 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f2490003be0 con 0x7f24a0094b10 2024-09-21T09:43:13.404 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.401+0000 7f248effd640 1 -- 172.21.15.92:0/4134367717 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(25..25 src has 1..25) ==== 3027+0+0 (secure 0 0 0) 0x7f249008d300 con 0x7f24a0094b10 2024-09-21T09:43:13.406 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.404+0000 7f248effd640 1 -- 172.21.15.92:0/4134367717 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f24a0092eb0 con 0x7f24a0094b10 2024-09-21T09:43:13.557 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.555+0000 7f24a4cb3640 1 -- 172.21.15.92:0/4134367717 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f24a012b870 con 0x7f24a0094b10 2024-09-21T09:43:13.557 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.556+0000 7f248effd640 1 -- 172.21.15.92:0/4134367717 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v25) ==== 74+0+8640 (secure 0 0 0) 0x7f249008c3b0 con 0x7f24a0094b10 2024-09-21T09:43:13.557 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:13.557 INFO:teuthology.orchestra.run.smithi092.stdout:{"epoch":25,"fsid":"5f250e6c-4e36-49b1-ab62-d9454369fa03","created":"2024-09-21T09:41:34.680338+0000","modified":"2024-09-21T09:43:13.277270+0000","last_up_change":"2024-09-21T09:43:13.277270+0000","last_in_change":"2024-09-21T09:41:37.598778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"squid","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:41:50.040313+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:41:52.054798+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"f3314f59-3374-4072-85c9-f8c4e458efc8","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6805","nonce":195102377}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6806","nonce":195102377}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6808","nonce":195102377}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6807","nonce":195102377}]},"public_addr":"172.21.15.92:6805/195102377","cluster_addr":"172.21.15.92:6806/195102377","heartbeat_back_addr":"172.21.15.92:6808/195102377","heartbeat_front_addr":"172.21.15.92:6807/195102377","state":["exists"]},{"osd":1,"uuid":"cf0486af-ba68-4763-a364-c1f2181604a7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6813","nonce":1359151798}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6814","nonce":1359151798}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6816","nonce":1359151798}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6815","nonce":1359151798}]},"public_addr":"172.21.15.92:6813/1359151798","cluster_addr":"172.21.15.92:6814/1359151798","heartbeat_back_addr":"172.21.15.92:6816/1359151798","heartbeat_front_addr":"172.21.15.92:6815/1359151798","state":["exists"]},{"osd":2,"uuid":"5c587421-3409-4469-a2ee-b2ea20cd45e5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6809","nonce":296401696}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6810","nonce":296401696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6812","nonce":296401696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6811","nonce":296401696}]},"public_addr":"172.21.15.92:6809/296401696","cluster_addr":"172.21.15.92:6810/296401696","heartbeat_back_addr":"172.21.15.92:6812/296401696","heartbeat_front_addr":"172.21.15.92:6811/296401696","state":["exists"]},{"osd":3,"uuid":"b98663b7-de4d-4c12-ac91-8c33f159da35","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":24,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6801","nonce":1829519295}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6802","nonce":1829519295}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6804","nonce":1829519295}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6803","nonce":1829519295}]},"public_addr":"172.21.15.92:6801/1829519295","cluster_addr":"172.21.15.92:6802/1829519295","heartbeat_back_addr":"172.21.15.92:6804/1829519295","heartbeat_front_addr":"172.21.15.92:6803/1829519295","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.934317+0000","dead_epoch":22},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:45.144020+0000","dead_epoch":23},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.895848+0000","dead_epoch":24},{"osd":3,"down_stamp":"2024-09-21T09:43:13.277270+0000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.636219+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.92:0/230938560":"2024-09-22T09:43:06.249869+0000","172.21.15.92:0/162460762":"2024-09-22T09:43:06.249869+0000","172.21.15.92:6800/1280404317":"2024-09-22T09:43:06.249869+0000","172.21.15.92:0/3587832392":"2024-09-22T09:43:06.249869+0000","172.21.15.92:0/2147030555":"2024-09-22T09:43:06.249869+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:43:13.560 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.558+0000 7f24a4cb3640 1 -- 172.21.15.92:0/4134367717 >> v1:172.21.15.92:6800/2299163855 conn(0x7f24800514e0 legacy=0x7f24800539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:13.560 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.558+0000 7f24a4cb3640 1 -- 172.21.15.92:0/4134367717 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f24a0094b10 msgr2=0x7f24a01304d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:13.560 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.558+0000 7f24a4cb3640 1 --2- 172.21.15.92:0/4134367717 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f24a0094b10 0x7f24a01304d0 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f2490009780 tx=0x7f2490033b70 comp rx=0 tx=0).stop 2024-09-21T09:43:13.560 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.558+0000 7f24a4cb3640 1 -- 172.21.15.92:0/4134367717 shutdown_connections 2024-09-21T09:43:13.560 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.558+0000 7f24a4cb3640 1 --2- 172.21.15.92:0/4134367717 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f24a012bdb0 0x7f24a01c2840 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:13.560 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.558+0000 7f24a4cb3640 1 --2- 172.21.15.92:0/4134367717 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f24a012b490 0x7f24a0130a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:13.560 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.558+0000 7f24a4cb3640 1 --2- 172.21.15.92:0/4134367717 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f24a0094b10 0x7f24a01304d0 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:13.560 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.558+0000 7f24a4cb3640 1 -- 172.21.15.92:0/4134367717 >> 172.21.15.92:0/4134367717 conn(0x7f24a009a2e0 msgr2=0x7f24a00922c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:13.560 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.558+0000 7f24a4cb3640 1 -- 172.21.15.92:0/4134367717 shutdown_connections 2024-09-21T09:43:13.560 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:13.558+0000 7f24a4cb3640 1 -- 172.21.15.92:0/4134367717 wait complete. 2024-09-21T09:43:13.704 INFO:tasks.ceph.osd.2.smithi092.stderr:2024-09-21T09:43:13.702+0000 7f32eb1bf640 -1 Falling back to public interface 2024-09-21T09:43:14.216 INFO:tasks.ceph.osd.3.smithi092.stderr:2024-09-21T09:43:14.214+0000 7fb217416640 -1 Falling back to public interface 2024-09-21T09:43:15.527 INFO:tasks.ceph.osd.0.smithi092.stderr:2024-09-21T09:43:15.525+0000 7f3a7140b640 -1 osd.0 21 log_to_monitors true 2024-09-21T09:43:16.309 INFO:tasks.ceph.osd.0.smithi092.stderr:2024-09-21T09:43:16.307+0000 7f3a64c19640 -1 osd.0 21 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:43:16.572 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:43:16.673 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.670+0000 7fea55eb3640 1 -- 172.21.15.92:0/2108272328 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fea5012fbe0 msgr2=0x7fea50131fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:16.673 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.670+0000 7fea55eb3640 1 --2- 172.21.15.92:0/2108272328 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fea5012fbe0 0x7fea50131fd0 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7fea38005fd0 tx=0x7fea3802a540 comp rx=0 tx=0).stop 2024-09-21T09:43:16.673 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.670+0000 7fea55eb3640 1 -- 172.21.15.92:0/2108272328 shutdown_connections 2024-09-21T09:43:16.673 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.670+0000 7fea55eb3640 1 --2- 172.21.15.92:0/2108272328 >> v2:172.21.15.92:3300/0 conn(0x7fea50132510 0x7fea50134900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:16.673 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.670+0000 7fea55eb3640 1 --2- 172.21.15.92:0/2108272328 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fea5012fbe0 0x7fea50131fd0 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:16.673 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.670+0000 7fea55eb3640 1 -- 172.21.15.92:0/2108272328 >> 172.21.15.92:0/2108272328 conn(0x7fea5009a2e0 msgr2=0x7fea50097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:16.673 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.670+0000 7fea55eb3640 1 -- 172.21.15.92:0/2108272328 shutdown_connections 2024-09-21T09:43:16.673 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.670+0000 7fea55eb3640 1 -- 172.21.15.92:0/2108272328 wait complete. 2024-09-21T09:43:16.673 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.671+0000 7fea55eb3640 1 Processor -- start 2024-09-21T09:43:16.673 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.671+0000 7fea55eb3640 1 -- start start 2024-09-21T09:43:16.675 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea55eb3640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fea5012cc30 0x7fea500910d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:16.676 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea55eb3640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fea5012fbe0 0x7fea50091610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:16.676 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea55eb3640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fea50132510 0x7fea50091b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:16.676 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea55eb3640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fea50135870 con 0x7fea5012cc30 2024-09-21T09:43:16.676 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea55eb3640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fea501356f0 con 0x7fea50132510 2024-09-21T09:43:16.676 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea55eb3640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fea501359f0 con 0x7fea5012fbe0 2024-09-21T09:43:16.676 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea4f7fe640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fea5012cc30 0x7fea500910d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:16.676 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea4ffff640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fea50132510 0x7fea50091b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:16.676 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea4f7fe640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fea5012cc30 0x7fea500910d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:50602/0 (socket says 172.21.15.92:50602) 2024-09-21T09:43:16.676 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea4f7fe640 1 -- 172.21.15.92:0/776646542 learned_addr learned my addr 172.21.15.92:0/776646542 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:16.676 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea46ffd640 1 --2- 172.21.15.92:0/776646542 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fea5012fbe0 0x7fea50091610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:16.676 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea4f7fe640 1 -- 172.21.15.92:0/776646542 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fea5012fbe0 msgr2=0x7fea50091610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:16.676 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea4f7fe640 1 --2- 172.21.15.92:0/776646542 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fea5012fbe0 0x7fea50091610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:16.676 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea4f7fe640 1 -- 172.21.15.92:0/776646542 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fea50132510 msgr2=0x7fea50091b50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:16.677 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea4f7fe640 1 --2- 172.21.15.92:0/776646542 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fea50132510 0x7fea50091b50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:16.677 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea4f7fe640 1 -- 172.21.15.92:0/776646542 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fea50092960 con 0x7fea5012cc30 2024-09-21T09:43:16.677 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea4ffff640 1 --2- 172.21.15.92:0/776646542 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fea50132510 0x7fea50091b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:16.677 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea4f7fe640 1 --2- 172.21.15.92:0/776646542 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fea5012cc30 0x7fea500910d0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fea4000c450 tx=0x7fea4000c850 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:16.677 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea4cff9640 1 -- 172.21.15.92:0/776646542 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fea40017070 con 0x7fea5012cc30 2024-09-21T09:43:16.677 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea4cff9640 1 -- 172.21.15.92:0/776646542 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fea400053c0 con 0x7fea5012cc30 2024-09-21T09:43:16.677 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea4cff9640 1 -- 172.21.15.92:0/776646542 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fea40003850 con 0x7fea5012cc30 2024-09-21T09:43:16.677 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea55eb3640 1 -- 172.21.15.92:0/776646542 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fea50126e50 con 0x7fea5012cc30 2024-09-21T09:43:16.677 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.672+0000 7fea55eb3640 1 -- 172.21.15.92:0/776646542 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fea50092540 con 0x7fea5012cc30 2024-09-21T09:43:16.677 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.673+0000 7fea55eb3640 1 -- 172.21.15.92:0/776646542 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fea5012d010 con 0x7fea5012cc30 2024-09-21T09:43:16.677 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.673+0000 7fea4cff9640 1 -- 172.21.15.92:0/776646542 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7fea4001a070 con 0x7fea5012cc30 2024-09-21T09:43:16.677 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.674+0000 7fea4cff9640 1 -- 172.21.15.92:0/776646542 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(26..26 src has 1..26) ==== 3027+0+0 (secure 0 0 0) 0x7fea400683d0 con 0x7fea5012cc30 2024-09-21T09:43:16.679 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.678+0000 7fea4cff9640 1 -- 172.21.15.92:0/776646542 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fea400039f0 con 0x7fea5012cc30 2024-09-21T09:43:16.836 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.833+0000 7fea55eb3640 1 -- 172.21.15.92:0/776646542 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fea500699c0 con 0x7fea5012cc30 2024-09-21T09:43:16.836 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.833+0000 7fea4cff9640 1 -- 172.21.15.92:0/776646542 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v26) ==== 74+0+8640 (secure 0 0 0) 0x7fea40067480 con 0x7fea5012cc30 2024-09-21T09:43:16.836 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:16.836 INFO:teuthology.orchestra.run.smithi092.stdout:{"epoch":26,"fsid":"5f250e6c-4e36-49b1-ab62-d9454369fa03","created":"2024-09-21T09:41:34.680338+0000","modified":"2024-09-21T09:43:16.293691+0000","last_up_change":"2024-09-21T09:43:13.277270+0000","last_in_change":"2024-09-21T09:41:37.598778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":6,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"squid","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:41:50.040313+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:41:52.054798+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"f3314f59-3374-4072-85c9-f8c4e458efc8","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":0,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6805","nonce":195102377}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6806","nonce":195102377}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6808","nonce":195102377}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6807","nonce":195102377}]},"public_addr":"172.21.15.92:6805/195102377","cluster_addr":"172.21.15.92:6806/195102377","heartbeat_back_addr":"172.21.15.92:6808/195102377","heartbeat_front_addr":"172.21.15.92:6807/195102377","state":["exists"]},{"osd":1,"uuid":"cf0486af-ba68-4763-a364-c1f2181604a7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6813","nonce":1359151798}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6814","nonce":1359151798}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6816","nonce":1359151798}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6815","nonce":1359151798}]},"public_addr":"172.21.15.92:6813/1359151798","cluster_addr":"172.21.15.92:6814/1359151798","heartbeat_back_addr":"172.21.15.92:6816/1359151798","heartbeat_front_addr":"172.21.15.92:6815/1359151798","state":["exists"]},{"osd":2,"uuid":"5c587421-3409-4469-a2ee-b2ea20cd45e5","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":11,"up_thru":15,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6809","nonce":296401696}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6810","nonce":296401696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6812","nonce":296401696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6811","nonce":296401696}]},"public_addr":"172.21.15.92:6809/296401696","cluster_addr":"172.21.15.92:6810/296401696","heartbeat_back_addr":"172.21.15.92:6812/296401696","heartbeat_front_addr":"172.21.15.92:6811/296401696","state":["exists"]},{"osd":3,"uuid":"b98663b7-de4d-4c12-ac91-8c33f159da35","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":24,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6801","nonce":1829519295}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6802","nonce":1829519295}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6804","nonce":1829519295}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6803","nonce":1829519295}]},"public_addr":"172.21.15.92:6801/1829519295","cluster_addr":"172.21.15.92:6802/1829519295","heartbeat_back_addr":"172.21.15.92:6804/1829519295","heartbeat_front_addr":"172.21.15.92:6803/1829519295","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.934317+0000","dead_epoch":22},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:45.144020+0000","dead_epoch":23},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.895848+0000","dead_epoch":24},{"osd":3,"down_stamp":"2024-09-21T09:43:13.277270+0000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.636219+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.92:0/230938560":"2024-09-22T09:43:06.249869+0000","172.21.15.92:0/162460762":"2024-09-22T09:43:06.249869+0000","172.21.15.92:6800/1280404317":"2024-09-22T09:43:06.249869+0000","172.21.15.92:0/3587832392":"2024-09-22T09:43:06.249869+0000","172.21.15.92:0/2147030555":"2024-09-22T09:43:06.249869+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:43:16.838 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.835+0000 7fea55eb3640 1 -- 172.21.15.92:0/776646542 >> v1:172.21.15.92:6800/2299163855 conn(0x7fea300514e0 legacy=0x7fea300539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:16.839 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.835+0000 7fea55eb3640 1 -- 172.21.15.92:0/776646542 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fea5012cc30 msgr2=0x7fea500910d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:16.839 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.835+0000 7fea55eb3640 1 --2- 172.21.15.92:0/776646542 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fea5012cc30 0x7fea500910d0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fea4000c450 tx=0x7fea4000c850 comp rx=0 tx=0).stop 2024-09-21T09:43:16.839 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.835+0000 7fea55eb3640 1 -- 172.21.15.92:0/776646542 shutdown_connections 2024-09-21T09:43:16.839 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.835+0000 7fea55eb3640 1 --2- 172.21.15.92:0/776646542 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fea50132510 0x7fea50091b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:16.839 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.835+0000 7fea55eb3640 1 --2- 172.21.15.92:0/776646542 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fea5012fbe0 0x7fea50091610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:16.839 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.835+0000 7fea55eb3640 1 --2- 172.21.15.92:0/776646542 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fea5012cc30 0x7fea500910d0 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:16.839 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.835+0000 7fea55eb3640 1 -- 172.21.15.92:0/776646542 >> 172.21.15.92:0/776646542 conn(0x7fea5009a2e0 msgr2=0x7fea5012d910 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:16.839 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.835+0000 7fea55eb3640 1 -- 172.21.15.92:0/776646542 shutdown_connections 2024-09-21T09:43:16.839 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:16.835+0000 7fea55eb3640 1 -- 172.21.15.92:0/776646542 wait complete. 2024-09-21T09:43:17.032 INFO:tasks.ceph.osd.1.smithi092.stderr:2024-09-21T09:43:17.029+0000 7f0791c64640 -1 osd.1 21 log_to_monitors true 2024-09-21T09:43:17.311 INFO:tasks.ceph.osd.1.smithi092.stderr:2024-09-21T09:43:17.308+0000 7f0785a13640 -1 osd.1 21 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:43:17.754 INFO:tasks.ceph.osd.2.smithi092.stderr:2024-09-21T09:43:17.751+0000 7f32eb1bf640 -1 osd.2 23 log_to_monitors true 2024-09-21T09:43:18.313 INFO:tasks.ceph.osd.2.smithi092.stderr:2024-09-21T09:43:18.309+0000 7f32def6e640 -1 osd.2 23 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:43:18.518 INFO:tasks.ceph.osd.3.smithi092.stderr:2024-09-21T09:43:18.515+0000 7fb217416640 -1 osd.3 24 log_to_monitors true 2024-09-21T09:43:19.313 INFO:tasks.ceph.osd.3.smithi092.stderr:2024-09-21T09:43:19.311+0000 7fb20ac00640 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:43:19.851 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:43:19.959 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.955+0000 7f910b339640 1 -- 172.21.15.92:0/1065985605 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f9104094ce0 msgr2=0x7f9104095130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:19.959 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.955+0000 7f910b339640 1 --2- 172.21.15.92:0/1065985605 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f9104094ce0 0x7f9104095130 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7f90fc005fd0 tx=0x7f90fc02a780 comp rx=0 tx=0).stop 2024-09-21T09:43:19.959 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.956+0000 7f910b339640 1 -- 172.21.15.92:0/1065985605 shutdown_connections 2024-09-21T09:43:19.959 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.956+0000 7f910b339640 1 --2- 172.21.15.92:0/1065985605 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f9104094ce0 0x7f9104095130 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:19.959 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.956+0000 7f910b339640 1 --2- 172.21.15.92:0/1065985605 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f910412b490 0x7f910412b870 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:19.959 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.956+0000 7f910b339640 1 -- 172.21.15.92:0/1065985605 >> 172.21.15.92:0/1065985605 conn(0x7f910409a2e0 msgr2=0x7f9104097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:19.959 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.956+0000 7f910b339640 1 -- 172.21.15.92:0/1065985605 shutdown_connections 2024-09-21T09:43:19.959 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.956+0000 7f910b339640 1 -- 172.21.15.92:0/1065985605 wait complete. 2024-09-21T09:43:19.962 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.956+0000 7f910b339640 1 Processor -- start 2024-09-21T09:43:19.962 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.957+0000 7f910b339640 1 -- start start 2024-09-21T09:43:19.962 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.957+0000 7f910b339640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f9104094ce0 0x7f910412c230 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:19.962 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.957+0000 7f910b339640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f910412b490 0x7f910412c770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:19.962 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.957+0000 7f910b339640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f910412bdb0 0x7f910412ccb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:19.962 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.957+0000 7f910b339640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f9104133600 con 0x7f9104094ce0 2024-09-21T09:43:19.962 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.957+0000 7f910b339640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f9104133480 con 0x7f910412bdb0 2024-09-21T09:43:19.962 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.957+0000 7f910b339640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f9104133780 con 0x7f910412b490 2024-09-21T09:43:19.962 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.957+0000 7f91090d5640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f9104094ce0 0x7f910412c230 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:19.962 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.957+0000 7f91098d6640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f910412bdb0 0x7f910412ccb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:19.962 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.957+0000 7f91088d4640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f910412b490 0x7f910412c770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:19.963 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.957+0000 7f91090d5640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f9104094ce0 0x7f910412c230 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:50618/0 (socket says 172.21.15.92:50618) 2024-09-21T09:43:19.963 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.957+0000 7f91098d6640 1 --2- 172.21.15.92:0/3791784178 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f910412bdb0 0x7f910412ccb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:43700/0 (socket says 172.21.15.92:43700) 2024-09-21T09:43:19.963 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.957+0000 7f91090d5640 1 -- 172.21.15.92:0/3791784178 learned_addr learned my addr 172.21.15.92:0/3791784178 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:19.963 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.957+0000 7f91090d5640 1 -- 172.21.15.92:0/3791784178 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f910412b490 msgr2=0x7f910412c770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:19.963 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.957+0000 7f91090d5640 1 --2- 172.21.15.92:0/3791784178 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f910412b490 0x7f910412c770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:19.963 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.957+0000 7f91090d5640 1 -- 172.21.15.92:0/3791784178 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f910412bdb0 msgr2=0x7f910412ccb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:19.963 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.957+0000 7f91090d5640 1 --2- 172.21.15.92:0/3791784178 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f910412bdb0 0x7f910412ccb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:19.963 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.957+0000 7f91090d5640 1 -- 172.21.15.92:0/3791784178 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f910412e430 con 0x7f9104094ce0 2024-09-21T09:43:19.963 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.957+0000 7f91098d6640 1 --2- 172.21.15.92:0/3791784178 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f910412bdb0 0x7f910412ccb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:19.963 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.958+0000 7f91090d5640 1 --2- 172.21.15.92:0/3791784178 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f9104094ce0 0x7f910412c230 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f910000ea60 tx=0x7f910000ef20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:19.963 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.958+0000 7f90f9ffb640 1 -- 172.21.15.92:0/3791784178 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f910000cde0 con 0x7f9104094ce0 2024-09-21T09:43:19.963 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.958+0000 7f90f9ffb640 1 -- 172.21.15.92:0/3791784178 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f9100004020 con 0x7f9104094ce0 2024-09-21T09:43:19.963 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.958+0000 7f910b339640 1 -- 172.21.15.92:0/3791784178 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9104090ac0 con 0x7f9104094ce0 2024-09-21T09:43:19.963 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.958+0000 7f90f9ffb640 1 -- 172.21.15.92:0/3791784178 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9100004450 con 0x7f9104094ce0 2024-09-21T09:43:19.964 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.958+0000 7f910b339640 1 -- 172.21.15.92:0/3791784178 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9104132e10 con 0x7f9104094ce0 2024-09-21T09:43:19.964 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.959+0000 7f90f9ffb640 1 -- 172.21.15.92:0/3791784178 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f91000035f0 con 0x7f9104094ce0 2024-09-21T09:43:19.964 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.959+0000 7f910b339640 1 -- 172.21.15.92:0/3791784178 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9104095130 con 0x7f9104094ce0 2024-09-21T09:43:19.964 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.960+0000 7f90f9ffb640 1 -- 172.21.15.92:0/3791784178 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(29..29 src has 1..29) ==== 3027+0+0 (secure 0 0 0) 0x7f9100069970 con 0x7f9104094ce0 2024-09-21T09:43:19.966 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:19.963+0000 7f90f9ffb640 1 -- 172.21.15.92:0/3791784178 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f910412e430 con 0x7f9104094ce0 2024-09-21T09:43:20.115 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:20.113+0000 7f910b339640 1 -- 172.21.15.92:0/3791784178 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f910412da30 con 0x7f9104094ce0 2024-09-21T09:43:20.116 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:20.113+0000 7f90f9ffb640 1 -- 172.21.15.92:0/3791784178 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v29) ==== 74+0+9145 (secure 0 0 0) 0x7f9100068a20 con 0x7f9104094ce0 2024-09-21T09:43:20.116 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:20.116 INFO:teuthology.orchestra.run.smithi092.stdout:{"epoch":29,"fsid":"5f250e6c-4e36-49b1-ab62-d9454369fa03","created":"2024-09-21T09:41:34.680338+0000","modified":"2024-09-21T09:43:19.302304+0000","last_up_change":"2024-09-21T09:43:19.302304+0000","last_in_change":"2024-09-21T09:41:37.598778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":9,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"squid","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:41:50.040313+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:41:52.054798+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":0.75,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f3314f59-3374-4072-85c9-f8c4e458efc8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":21,"up_from":27,"up_thru":27,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6805","nonce":7087154}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6806","nonce":7087154}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6808","nonce":7087154}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6807","nonce":7087154}]},"public_addr":"172.21.15.92:6805/7087154","cluster_addr":"172.21.15.92:6806/7087154","heartbeat_back_addr":"172.21.15.92:6808/7087154","heartbeat_front_addr":"172.21.15.92:6807/7087154","state":["exists","up"]},{"osd":1,"uuid":"cf0486af-ba68-4763-a364-c1f2181604a7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":21,"up_from":28,"up_thru":28,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6801","nonce":2577383826}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6802","nonce":2577383826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6804","nonce":2577383826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6803","nonce":2577383826}]},"public_addr":"172.21.15.92:6801/2577383826","cluster_addr":"172.21.15.92:6802/2577383826","heartbeat_back_addr":"172.21.15.92:6804/2577383826","heartbeat_front_addr":"172.21.15.92:6803/2577383826","state":["exists","up"]},{"osd":2,"uuid":"5c587421-3409-4469-a2ee-b2ea20cd45e5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":23,"up_from":29,"up_thru":15,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6809","nonce":4235634095}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6810","nonce":4235634095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6812","nonce":4235634095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6811","nonce":4235634095}]},"public_addr":"172.21.15.92:6809/4235634095","cluster_addr":"172.21.15.92:6810/4235634095","heartbeat_back_addr":"172.21.15.92:6812/4235634095","heartbeat_front_addr":"172.21.15.92:6811/4235634095","state":["exists","up"]},{"osd":3,"uuid":"b98663b7-de4d-4c12-ac91-8c33f159da35","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":24,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6801","nonce":1829519295}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6802","nonce":1829519295}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6804","nonce":1829519295}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6803","nonce":1829519295}]},"public_addr":"172.21.15.92:6801/1829519295","cluster_addr":"172.21.15.92:6802/1829519295","heartbeat_back_addr":"172.21.15.92:6804/1829519295","heartbeat_front_addr":"172.21.15.92:6803/1829519295","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.934317+0000","dead_epoch":22},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:45.144020+0000","dead_epoch":23},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.895848+0000","dead_epoch":24},{"osd":3,"down_stamp":"2024-09-21T09:43:13.277270+0000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.636219+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.92:0/230938560":"2024-09-22T09:43:06.249869+0000","172.21.15.92:0/162460762":"2024-09-22T09:43:06.249869+0000","172.21.15.92:6800/1280404317":"2024-09-22T09:43:06.249869+0000","172.21.15.92:0/3587832392":"2024-09-22T09:43:06.249869+0000","172.21.15.92:0/2147030555":"2024-09-22T09:43:06.249869+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:43:20.118 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:20.115+0000 7f910b339640 1 -- 172.21.15.92:0/3791784178 >> v1:172.21.15.92:6800/2299163855 conn(0x7f90e80514b0 legacy=0x7f90e8053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:20.118 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:20.115+0000 7f910b339640 1 -- 172.21.15.92:0/3791784178 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f9104094ce0 msgr2=0x7f910412c230 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:20.118 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:20.115+0000 7f910b339640 1 --2- 172.21.15.92:0/3791784178 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f9104094ce0 0x7f910412c230 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f910000ea60 tx=0x7f910000ef20 comp rx=0 tx=0).stop 2024-09-21T09:43:20.118 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:20.115+0000 7f910b339640 1 -- 172.21.15.92:0/3791784178 shutdown_connections 2024-09-21T09:43:20.118 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:20.115+0000 7f910b339640 1 --2- 172.21.15.92:0/3791784178 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f910412bdb0 0x7f910412ccb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:20.118 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:20.115+0000 7f910b339640 1 --2- 172.21.15.92:0/3791784178 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f910412b490 0x7f910412c770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:20.118 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:20.115+0000 7f910b339640 1 --2- 172.21.15.92:0/3791784178 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f9104094ce0 0x7f910412c230 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:20.118 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:20.115+0000 7f910b339640 1 -- 172.21.15.92:0/3791784178 >> 172.21.15.92:0/3791784178 conn(0x7f910409a2e0 msgr2=0x7f910412a630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:20.119 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:20.115+0000 7f910b339640 1 -- 172.21.15.92:0/3791784178 shutdown_connections 2024-09-21T09:43:20.119 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:20.115+0000 7f910b339640 1 -- 172.21.15.92:0/3791784178 wait complete. 2024-09-21T09:43:23.129 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:43:23.245 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.242+0000 7fe84866d640 1 -- 172.21.15.92:0/342603780 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe840132510 msgr2=0x7fe840134900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:23.246 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.242+0000 7fe84866d640 1 --2- 172.21.15.92:0/342603780 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe840132510 0x7fe840134900 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7fe838009830 tx=0x7fe83802a480 comp rx=0 tx=0).stop 2024-09-21T09:43:23.246 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.242+0000 7fe84866d640 1 -- 172.21.15.92:0/342603780 shutdown_connections 2024-09-21T09:43:23.246 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.242+0000 7fe84866d640 1 --2- 172.21.15.92:0/342603780 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe840132510 0x7fe840134900 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:23.246 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.242+0000 7fe84866d640 1 --2- 172.21.15.92:0/342603780 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe84012fbe0 0x7fe840131fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:23.246 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.242+0000 7fe84866d640 1 -- 172.21.15.92:0/342603780 >> 172.21.15.92:0/342603780 conn(0x7fe84009a2e0 msgr2=0x7fe840097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:23.246 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.242+0000 7fe84866d640 1 -- 172.21.15.92:0/342603780 shutdown_connections 2024-09-21T09:43:23.246 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.242+0000 7fe84866d640 1 -- 172.21.15.92:0/342603780 wait complete. 2024-09-21T09:43:23.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.243+0000 7fe84866d640 1 Processor -- start 2024-09-21T09:43:23.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.243+0000 7fe84866d640 1 -- start start 2024-09-21T09:43:23.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.244+0000 7fe84866d640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe84012cc30 0x7fe840127bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:23.248 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.244+0000 7fe84866d640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe84012fbe0 0x7fe840128100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:23.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.244+0000 7fe84866d640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe840132510 0x7fe840128640 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:23.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.244+0000 7fe84866d640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fe840135870 con 0x7fe840132510 2024-09-21T09:43:23.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.244+0000 7fe84866d640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fe8401356f0 con 0x7fe84012cc30 2024-09-21T09:43:23.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.244+0000 7fe84866d640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fe8401359f0 con 0x7fe84012fbe0 2024-09-21T09:43:23.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.244+0000 7fe846409640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe84012cc30 0x7fe840127bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:23.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.244+0000 7fe846c0a640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe840132510 0x7fe840128640 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:23.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.244+0000 7fe845c08640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe84012fbe0 0x7fe840128100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:23.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.244+0000 7fe846409640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe84012cc30 0x7fe840127bc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:43712/0 (socket says 172.21.15.92:43712) 2024-09-21T09:43:23.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.244+0000 7fe846409640 1 -- 172.21.15.92:0/1657188186 learned_addr learned my addr 172.21.15.92:0/1657188186 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:23.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.244+0000 7fe846c0a640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe840132510 0x7fe840128640 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:50636/0 (socket says 172.21.15.92:50636) 2024-09-21T09:43:23.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.244+0000 7fe846409640 1 -- 172.21.15.92:0/1657188186 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe84012fbe0 msgr2=0x7fe840128100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:23.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.244+0000 7fe846409640 1 --2- 172.21.15.92:0/1657188186 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe84012fbe0 0x7fe840128100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:23.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.244+0000 7fe846409640 1 -- 172.21.15.92:0/1657188186 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe840132510 msgr2=0x7fe840128640 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:23.249 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.244+0000 7fe846409640 1 --2- 172.21.15.92:0/1657188186 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe840132510 0x7fe840128640 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:23.250 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.244+0000 7fe846409640 1 -- 172.21.15.92:0/1657188186 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe840122e70 con 0x7fe84012cc30 2024-09-21T09:43:23.250 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.244+0000 7fe845c08640 1 --2- 172.21.15.92:0/1657188186 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe84012fbe0 0x7fe840128100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:23.250 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.244+0000 7fe846409640 1 --2- 172.21.15.92:0/1657188186 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe84012cc30 0x7fe840127bc0 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7fe83000c820 tx=0x7fe83000cce0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:23.250 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.244+0000 7fe836ffd640 1 -- 172.21.15.92:0/1657188186 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe830003da0 con 0x7fe84012cc30 2024-09-21T09:43:23.250 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.245+0000 7fe836ffd640 1 -- 172.21.15.92:0/1657188186 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fe830005070 con 0x7fe84012cc30 2024-09-21T09:43:23.250 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.245+0000 7fe84866d640 1 -- 172.21.15.92:0/1657188186 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe840129d00 con 0x7fe84012cc30 2024-09-21T09:43:23.250 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.245+0000 7fe836ffd640 1 -- 172.21.15.92:0/1657188186 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe8300054c0 con 0x7fe84012cc30 2024-09-21T09:43:23.250 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.245+0000 7fe84866d640 1 -- 172.21.15.92:0/1657188186 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fe840126e50 con 0x7fe84012cc30 2024-09-21T09:43:23.250 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.245+0000 7fe84866d640 1 -- 172.21.15.92:0/1657188186 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe84012d010 con 0x7fe84012cc30 2024-09-21T09:43:23.250 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.246+0000 7fe836ffd640 1 -- 172.21.15.92:0/1657188186 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7fe830005660 con 0x7fe84012cc30 2024-09-21T09:43:23.251 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.246+0000 7fe836ffd640 1 -- 172.21.15.92:0/1657188186 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(31..31 src has 1..31) ==== 3027+0+0 (secure 0 0 0) 0x7fe830068de0 con 0x7fe84012cc30 2024-09-21T09:43:23.253 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.250+0000 7fe836ffd640 1 -- 172.21.15.92:0/1657188186 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fe830067e90 con 0x7fe84012cc30 2024-09-21T09:43:23.406 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.403+0000 7fe84866d640 1 -- 172.21.15.92:0/1657188186 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fe8400699c0 con 0x7fe84012cc30 2024-09-21T09:43:23.407 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.404+0000 7fe836ffd640 1 -- 172.21.15.92:0/1657188186 <== mon.1 v2:172.21.15.92:3301/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v31) ==== 74+0+9145 (secure 0 0 0) 0x7fe8300679d0 con 0x7fe84012cc30 2024-09-21T09:43:23.407 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:23.407 INFO:teuthology.orchestra.run.smithi092.stdout:{"epoch":31,"fsid":"5f250e6c-4e36-49b1-ab62-d9454369fa03","created":"2024-09-21T09:41:34.680338+0000","modified":"2024-09-21T09:43:21.305027+0000","last_up_change":"2024-09-21T09:43:20.303761+0000","last_in_change":"2024-09-21T09:41:37.598778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":9,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"squid","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:41:50.040313+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:41:52.054798+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"18","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f3314f59-3374-4072-85c9-f8c4e458efc8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":21,"up_from":27,"up_thru":27,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6805","nonce":7087154}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6806","nonce":7087154}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6808","nonce":7087154}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6807","nonce":7087154}]},"public_addr":"172.21.15.92:6805/7087154","cluster_addr":"172.21.15.92:6806/7087154","heartbeat_back_addr":"172.21.15.92:6808/7087154","heartbeat_front_addr":"172.21.15.92:6807/7087154","state":["exists","up"]},{"osd":1,"uuid":"cf0486af-ba68-4763-a364-c1f2181604a7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":21,"up_from":28,"up_thru":28,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6801","nonce":2577383826}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6802","nonce":2577383826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6804","nonce":2577383826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6803","nonce":2577383826}]},"public_addr":"172.21.15.92:6801/2577383826","cluster_addr":"172.21.15.92:6802/2577383826","heartbeat_back_addr":"172.21.15.92:6804/2577383826","heartbeat_front_addr":"172.21.15.92:6803/2577383826","state":["exists","up"]},{"osd":2,"uuid":"5c587421-3409-4469-a2ee-b2ea20cd45e5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":23,"up_from":29,"up_thru":30,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6809","nonce":4235634095}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6810","nonce":4235634095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6812","nonce":4235634095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6811","nonce":4235634095}]},"public_addr":"172.21.15.92:6809/4235634095","cluster_addr":"172.21.15.92:6810/4235634095","heartbeat_back_addr":"172.21.15.92:6812/4235634095","heartbeat_front_addr":"172.21.15.92:6811/4235634095","state":["exists","up"]},{"osd":3,"uuid":"b98663b7-de4d-4c12-ac91-8c33f159da35","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":24,"up_from":30,"up_thru":30,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6813","nonce":138232475}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6814","nonce":138232475}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6816","nonce":138232475}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6815","nonce":138232475}]},"public_addr":"172.21.15.92:6813/138232475","cluster_addr":"172.21.15.92:6814/138232475","heartbeat_back_addr":"172.21.15.92:6816/138232475","heartbeat_front_addr":"172.21.15.92:6815/138232475","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.934317+0000","dead_epoch":22},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:45.144020+0000","dead_epoch":23},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.895848+0000","dead_epoch":24},{"osd":3,"down_stamp":"2024-09-21T09:43:13.277270+0000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.636219+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.92:0/230938560":"2024-09-22T09:43:06.249869+0000","172.21.15.92:0/162460762":"2024-09-22T09:43:06.249869+0000","172.21.15.92:6800/1280404317":"2024-09-22T09:43:06.249869+0000","172.21.15.92:0/3587832392":"2024-09-22T09:43:06.249869+0000","172.21.15.92:0/2147030555":"2024-09-22T09:43:06.249869+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:43:23.409 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.406+0000 7fe84866d640 1 -- 172.21.15.92:0/1657188186 >> v1:172.21.15.92:6800/2299163855 conn(0x7fe82c0514e0 legacy=0x7fe82c0539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:23.409 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.406+0000 7fe84866d640 1 -- 172.21.15.92:0/1657188186 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe84012cc30 msgr2=0x7fe840127bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:23.409 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.406+0000 7fe84866d640 1 --2- 172.21.15.92:0/1657188186 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe84012cc30 0x7fe840127bc0 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7fe83000c820 tx=0x7fe83000cce0 comp rx=0 tx=0).stop 2024-09-21T09:43:23.409 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.406+0000 7fe84866d640 1 -- 172.21.15.92:0/1657188186 shutdown_connections 2024-09-21T09:43:23.409 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.406+0000 7fe84866d640 1 --2- 172.21.15.92:0/1657188186 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe840132510 0x7fe840128640 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:23.409 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.406+0000 7fe84866d640 1 --2- 172.21.15.92:0/1657188186 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe84012fbe0 0x7fe840128100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:23.409 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.406+0000 7fe84866d640 1 --2- 172.21.15.92:0/1657188186 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe84012cc30 0x7fe840127bc0 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:23.410 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.406+0000 7fe84866d640 1 -- 172.21.15.92:0/1657188186 >> 172.21.15.92:0/1657188186 conn(0x7fe84009a2e0 msgr2=0x7fe840097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:23.410 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.406+0000 7fe84866d640 1 -- 172.21.15.92:0/1657188186 shutdown_connections 2024-09-21T09:43:23.410 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.406+0000 7fe84866d640 1 -- 172.21.15.92:0/1657188186 wait complete. 2024-09-21T09:43:23.420 INFO:tasks.ceph.ceph_manager.ceph:all up! 2024-09-21T09:43:23.420 INFO:teuthology.run_tasks:Running task exec... 2024-09-21T09:43:23.436 INFO:teuthology.task.exec:Executing custom commands... 2024-09-21T09:43:23.436 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi092.front.sepia.ceph.com 2024-09-21T09:43:23.436 DEBUG:teuthology.orchestra.run.smithi092:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph versions' 2024-09-21T09:43:23.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.530+0000 7fa1eafd5640 1 -- 172.21.15.92:0/2871462931 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fa1e409b3b0 msgr2=0x7fa1e406cb70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:23.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.530+0000 7fa1eafd5640 1 --2- 172.21.15.92:0/2871462931 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fa1e409b3b0 0x7fa1e406cb70 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7fa1d8009830 tx=0x7fa1d802a3f0 comp rx=0 tx=0).stop 2024-09-21T09:43:23.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.530+0000 7fa1eafd5640 1 -- 172.21.15.92:0/2871462931 shutdown_connections 2024-09-21T09:43:23.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.530+0000 7fa1eafd5640 1 --2- 172.21.15.92:0/2871462931 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fa1e406d0b0 0x7fa1e406d4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:23.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.530+0000 7fa1eafd5640 1 --2- 172.21.15.92:0/2871462931 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fa1e409b3b0 0x7fa1e406cb70 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:23.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.530+0000 7fa1eafd5640 1 -- 172.21.15.92:0/2871462931 >> 172.21.15.92:0/2871462931 conn(0x7fa1e409e7d0 msgr2=0x7fa1e409aca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:23.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.530+0000 7fa1eafd5640 1 -- 172.21.15.92:0/2871462931 shutdown_connections 2024-09-21T09:43:23.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.530+0000 7fa1eafd5640 1 -- 172.21.15.92:0/2871462931 wait complete. 2024-09-21T09:43:23.535 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.531+0000 7fa1eafd5640 1 Processor -- start 2024-09-21T09:43:23.535 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.531+0000 7fa1eafd5640 1 -- start start 2024-09-21T09:43:23.535 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.531+0000 7fa1eafd5640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fa1e406d0b0 0x7fa1e41301c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:23.535 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.531+0000 7fa1eafd5640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fa1e409b3b0 0x7fa1e4130700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:23.535 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.531+0000 7fa1eafd5640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fa1e4131320 0x7fa1e4130c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:23.536 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.531+0000 7fa1eafd5640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fa1e4135670 con 0x7fa1e4131320 2024-09-21T09:43:23.536 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.531+0000 7fa1eafd5640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fa1e41354f0 con 0x7fa1e409b3b0 2024-09-21T09:43:23.536 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.531+0000 7fa1eafd5640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fa1e41357f0 con 0x7fa1e406d0b0 2024-09-21T09:43:23.536 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.531+0000 7fa1e3fff640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fa1e409b3b0 0x7fa1e4130700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:23.536 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.531+0000 7fa1e8d71640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fa1e406d0b0 0x7fa1e41301c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:23.536 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.531+0000 7fa1e9572640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fa1e4131320 0x7fa1e4130c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:23.536 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.531+0000 7fa1e3fff640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fa1e409b3b0 0x7fa1e4130700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:43736/0 (socket says 172.21.15.92:43736) 2024-09-21T09:43:23.536 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.531+0000 7fa1e8d71640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fa1e406d0b0 0x7fa1e41301c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:42252/0 (socket says 172.21.15.92:42252) 2024-09-21T09:43:23.536 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.531+0000 7fa1e3fff640 1 -- 172.21.15.92:0/2338205375 learned_addr learned my addr 172.21.15.92:0/2338205375 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:23.536 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.532+0000 7fa1e3fff640 1 -- 172.21.15.92:0/2338205375 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fa1e406d0b0 msgr2=0x7fa1e41301c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:23.536 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.532+0000 7fa1e3fff640 1 --2- 172.21.15.92:0/2338205375 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fa1e406d0b0 0x7fa1e41301c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:23.536 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.532+0000 7fa1e3fff640 1 -- 172.21.15.92:0/2338205375 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fa1e4131320 msgr2=0x7fa1e4130c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:23.536 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.532+0000 7fa1e3fff640 1 --2- 172.21.15.92:0/2338205375 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fa1e4131320 0x7fa1e4130c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:23.536 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.532+0000 7fa1e3fff640 1 -- 172.21.15.92:0/2338205375 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa1e412a210 con 0x7fa1e409b3b0 2024-09-21T09:43:23.537 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.532+0000 7fa1e3fff640 1 --2- 172.21.15.92:0/2338205375 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fa1e409b3b0 0x7fa1e4130700 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7fa1d000e9c0 tx=0x7fa1d000ee80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:23.537 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.532+0000 7fa1e17fa640 1 -- 172.21.15.92:0/2338205375 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa1d000cd80 con 0x7fa1e409b3b0 2024-09-21T09:43:23.537 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.532+0000 7fa1e17fa640 1 -- 172.21.15.92:0/2338205375 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fa1d0004020 con 0x7fa1e409b3b0 2024-09-21T09:43:23.537 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.532+0000 7fa1eafd5640 1 -- 172.21.15.92:0/2338205375 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa1e412eee0 con 0x7fa1e409b3b0 2024-09-21T09:43:23.537 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.532+0000 7fa1e17fa640 1 -- 172.21.15.92:0/2338205375 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa1d0004450 con 0x7fa1e409b3b0 2024-09-21T09:43:23.537 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.532+0000 7fa1eafd5640 1 -- 172.21.15.92:0/2338205375 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fa1e412f530 con 0x7fa1e409b3b0 2024-09-21T09:43:23.537 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.533+0000 7fa1eafd5640 1 -- 172.21.15.92:0/2338205375 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa1e406d4f0 con 0x7fa1e409b3b0 2024-09-21T09:43:23.537 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.533+0000 7fa1e17fa640 1 -- 172.21.15.92:0/2338205375 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7fa1d00035f0 con 0x7fa1e409b3b0 2024-09-21T09:43:23.537 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.533+0000 7fa1e17fa640 1 -- 172.21.15.92:0/2338205375 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(31..31 src has 1..31) ==== 3027+0+0 (secure 0 0 0) 0x7fa1d0069920 con 0x7fa1e409b3b0 2024-09-21T09:43:23.539 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.537+0000 7fa1e17fa640 1 -- 172.21.15.92:0/2338205375 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fa1d00689d0 con 0x7fa1e409b3b0 2024-09-21T09:43:23.742 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.740+0000 7fa1eafd5640 1 -- 172.21.15.92:0/2338205375 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "versions"} v 0) -- 0x7fa1e4068ad0 con 0x7fa1e409b3b0 2024-09-21T09:43:23.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.740+0000 7fa1e17fa640 1 -- 172.21.15.92:0/2338205375 <== mon.1 v2:172.21.15.92:3301/0 7 ==== mon_command_ack([{"prefix": "versions"}]=0 v0) ==== 56+0+499 (secure 0 0 0) 0x7fa1d0009460 con 0x7fa1e409b3b0 2024-09-21T09:43:23.743 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2024-09-21T09:43:23.743 INFO:teuthology.orchestra.run.smithi092.stdout: "mon": { 2024-09-21T09:43:23.743 INFO:teuthology.orchestra.run.smithi092.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:43:23.743 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:23.743 INFO:teuthology.orchestra.run.smithi092.stdout: "mgr": { 2024-09-21T09:43:23.743 INFO:teuthology.orchestra.run.smithi092.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:43:23.743 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:23.743 INFO:teuthology.orchestra.run.smithi092.stdout: "osd": { 2024-09-21T09:43:23.743 INFO:teuthology.orchestra.run.smithi092.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 4 2024-09-21T09:43:23.744 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:23.744 INFO:teuthology.orchestra.run.smithi092.stdout: "overall": { 2024-09-21T09:43:23.744 INFO:teuthology.orchestra.run.smithi092.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 9 2024-09-21T09:43:23.744 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:23.744 INFO:teuthology.orchestra.run.smithi092.stdout:} 2024-09-21T09:43:23.745 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.742+0000 7fa1eafd5640 1 -- 172.21.15.92:0/2338205375 >> v1:172.21.15.92:6800/2299163855 conn(0x7fa1c80514b0 legacy=0x7fa1c8053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:23.745 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.742+0000 7fa1eafd5640 1 -- 172.21.15.92:0/2338205375 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fa1e409b3b0 msgr2=0x7fa1e4130700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:23.745 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.742+0000 7fa1eafd5640 1 --2- 172.21.15.92:0/2338205375 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fa1e409b3b0 0x7fa1e4130700 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7fa1d000e9c0 tx=0x7fa1d000ee80 comp rx=0 tx=0).stop 2024-09-21T09:43:23.745 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.742+0000 7fa1eafd5640 1 -- 172.21.15.92:0/2338205375 shutdown_connections 2024-09-21T09:43:23.745 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.742+0000 7fa1eafd5640 1 --2- 172.21.15.92:0/2338205375 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fa1e4131320 0x7fa1e4130c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:23.745 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.742+0000 7fa1eafd5640 1 --2- 172.21.15.92:0/2338205375 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fa1e409b3b0 0x7fa1e4130700 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:23.745 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.742+0000 7fa1eafd5640 1 --2- 172.21.15.92:0/2338205375 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fa1e406d0b0 0x7fa1e41301c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:23.745 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.742+0000 7fa1eafd5640 1 -- 172.21.15.92:0/2338205375 >> 172.21.15.92:0/2338205375 conn(0x7fa1e409e7d0 msgr2=0x7fa1e4098ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:23.745 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.742+0000 7fa1eafd5640 1 -- 172.21.15.92:0/2338205375 shutdown_connections 2024-09-21T09:43:23.745 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.742+0000 7fa1eafd5640 1 -- 172.21.15.92:0/2338205375 wait complete. 2024-09-21T09:43:23.755 DEBUG:teuthology.orchestra.run.smithi092:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph osd dump -f json-pretty' 2024-09-21T09:43:23.853 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.850+0000 7f705481d640 1 -- 172.21.15.92:0/1065840776 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7050093b40 msgr2=0x7f70501333f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:23.853 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.850+0000 7f705481d640 1 --2- 172.21.15.92:0/1065840776 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7050093b40 0x7f70501333f0 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7f7048009be0 tx=0x7f704802a770 comp rx=0 tx=0).stop 2024-09-21T09:43:23.853 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.850+0000 7f705481d640 1 -- 172.21.15.92:0/1065840776 shutdown_connections 2024-09-21T09:43:23.853 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.850+0000 7f705481d640 1 --2- 172.21.15.92:0/1065840776 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7050093b40 0x7f70501333f0 unknown :-1 s=CLOSED pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:23.853 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.850+0000 7f705481d640 1 --2- 172.21.15.92:0/1065840776 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7050094200 0x7f7050130ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:23.854 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.850+0000 7f705481d640 1 -- 172.21.15.92:0/1065840776 >> 172.21.15.92:0/1065840776 conn(0x7f705009b580 msgr2=0x7f705009b990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:23.854 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.850+0000 7f705481d640 1 -- 172.21.15.92:0/1065840776 shutdown_connections 2024-09-21T09:43:23.854 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.850+0000 7f705481d640 1 -- 172.21.15.92:0/1065840776 wait complete. 2024-09-21T09:43:23.855 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.851+0000 7f705481d640 1 Processor -- start 2024-09-21T09:43:23.856 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.851+0000 7f705481d640 1 -- start start 2024-09-21T09:43:23.856 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.851+0000 7f705481d640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7050093050 0x7f7050070560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:23.856 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.851+0000 7f705481d640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7050093b40 0x7f7050070aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:23.856 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.851+0000 7f705481d640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7050094200 0x7f7050070fe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:23.856 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.851+0000 7f705481d640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f7050134720 con 0x7f7050093050 2024-09-21T09:43:23.856 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.851+0000 7f705481d640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f70501345a0 con 0x7f7050094200 2024-09-21T09:43:23.856 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.851+0000 7f705481d640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f70501348a0 con 0x7f7050093b40 2024-09-21T09:43:23.856 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.852+0000 7f7045d9b640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7050093b40 0x7f7050070aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:23.856 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.852+0000 7f704ed9d640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7050094200 0x7f7050070fe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:23.856 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.852+0000 7f7045d9b640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7050093b40 0x7f7050070aa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:42268/0 (socket says 172.21.15.92:42268) 2024-09-21T09:43:23.856 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.852+0000 7f7045d9b640 1 -- 172.21.15.92:0/959653741 learned_addr learned my addr 172.21.15.92:0/959653741 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:23.856 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.852+0000 7f704e59c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7050093050 0x7f7050070560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:23.856 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.852+0000 7f7045d9b640 1 -- 172.21.15.92:0/959653741 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7050094200 msgr2=0x7f7050070fe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:23.857 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.852+0000 7f7045d9b640 1 --2- 172.21.15.92:0/959653741 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7050094200 0x7f7050070fe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:23.857 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.852+0000 7f7045d9b640 1 -- 172.21.15.92:0/959653741 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7050093050 msgr2=0x7f7050070560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:23.857 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.852+0000 7f7045d9b640 1 --2- 172.21.15.92:0/959653741 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7050093050 0x7f7050070560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:23.857 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.852+0000 7f7045d9b640 1 -- 172.21.15.92:0/959653741 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f70500976c0 con 0x7f7050093b40 2024-09-21T09:43:23.857 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.852+0000 7f704e59c640 1 --2- 172.21.15.92:0/959653741 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7050093050 0x7f7050070560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:23.857 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.852+0000 7f7045d9b640 1 --2- 172.21.15.92:0/959653741 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7050093b40 0x7f7050070aa0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f703800ea60 tx=0x7f703800ef20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:23.857 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.852+0000 7f70477fe640 1 -- 172.21.15.92:0/959653741 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f703800cde0 con 0x7f7050093b40 2024-09-21T09:43:23.857 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.852+0000 7f70477fe640 1 -- 172.21.15.92:0/959653741 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f7038004020 con 0x7f7050093b40 2024-09-21T09:43:23.857 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.852+0000 7f705481d640 1 -- 172.21.15.92:0/959653741 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7050073d00 con 0x7f7050093b40 2024-09-21T09:43:23.857 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.852+0000 7f70477fe640 1 -- 172.21.15.92:0/959653741 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7038004450 con 0x7f7050093b40 2024-09-21T09:43:23.857 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.852+0000 7f705481d640 1 -- 172.21.15.92:0/959653741 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f7050074610 con 0x7f7050093b40 2024-09-21T09:43:23.858 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.853+0000 7f70477fe640 1 -- 172.21.15.92:0/959653741 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f70380035f0 con 0x7f7050093b40 2024-09-21T09:43:23.858 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.853+0000 7f705481d640 1 -- 172.21.15.92:0/959653741 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7050074870 con 0x7f7050093b40 2024-09-21T09:43:23.858 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.854+0000 7f70477fe640 1 -- 172.21.15.92:0/959653741 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(31..31 src has 1..31) ==== 3027+0+0 (secure 0 0 0) 0x7f7038069970 con 0x7f7050093b40 2024-09-21T09:43:23.860 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:23.857+0000 7f70477fe640 1 -- 172.21.15.92:0/959653741 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f7038068a20 con 0x7f7050093b40 2024-09-21T09:43:24.010 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.007+0000 7f705481d640 1 -- 172.21.15.92:0/959653741 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "osd dump", "format": "json-pretty"} v 0) -- 0x7f7050093430 con 0x7f7050093b40 2024-09-21T09:43:24.010 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.008+0000 7f70477fe640 1 -- 172.21.15.92:0/959653741 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json-pretty"}]=0 v31) ==== 81+0+16809 (secure 0 0 0) 0x7f7038068560 con 0x7f7050093b40 2024-09-21T09:43:24.011 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:24.011 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2024-09-21T09:43:24.011 INFO:teuthology.orchestra.run.smithi092.stdout: "epoch": 31, 2024-09-21T09:43:24.011 INFO:teuthology.orchestra.run.smithi092.stdout: "fsid": "5f250e6c-4e36-49b1-ab62-d9454369fa03", 2024-09-21T09:43:24.011 INFO:teuthology.orchestra.run.smithi092.stdout: "created": "2024-09-21T09:41:34.680338+0000", 2024-09-21T09:43:24.011 INFO:teuthology.orchestra.run.smithi092.stdout: "modified": "2024-09-21T09:43:21.305027+0000", 2024-09-21T09:43:24.011 INFO:teuthology.orchestra.run.smithi092.stdout: "last_up_change": "2024-09-21T09:43:20.303761+0000", 2024-09-21T09:43:24.011 INFO:teuthology.orchestra.run.smithi092.stdout: "last_in_change": "2024-09-21T09:41:37.598778+0000", 2024-09-21T09:43:24.011 INFO:teuthology.orchestra.run.smithi092.stdout: "flags": "sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit", 2024-09-21T09:43:24.011 INFO:teuthology.orchestra.run.smithi092.stdout: "flags_num": 5799936, 2024-09-21T09:43:24.011 INFO:teuthology.orchestra.run.smithi092.stdout: "flags_set": [ 2024-09-21T09:43:24.011 INFO:teuthology.orchestra.run.smithi092.stdout: "pglog_hardlimit", 2024-09-21T09:43:24.012 INFO:teuthology.orchestra.run.smithi092.stdout: "purged_snapdirs", 2024-09-21T09:43:24.012 INFO:teuthology.orchestra.run.smithi092.stdout: "recovery_deletes", 2024-09-21T09:43:24.012 INFO:teuthology.orchestra.run.smithi092.stdout: "sortbitwise" 2024-09-21T09:43:24.012 INFO:teuthology.orchestra.run.smithi092.stdout: ], 2024-09-21T09:43:24.012 INFO:teuthology.orchestra.run.smithi092.stdout: "crush_version": 9, 2024-09-21T09:43:24.012 INFO:teuthology.orchestra.run.smithi092.stdout: "full_ratio": 0.94999998807907104, 2024-09-21T09:43:24.012 INFO:teuthology.orchestra.run.smithi092.stdout: "backfillfull_ratio": 0.89999997615814209, 2024-09-21T09:43:24.012 INFO:teuthology.orchestra.run.smithi092.stdout: "nearfull_ratio": 0.85000002384185791, 2024-09-21T09:43:24.012 INFO:teuthology.orchestra.run.smithi092.stdout: "cluster_snapshot": "", 2024-09-21T09:43:24.012 INFO:teuthology.orchestra.run.smithi092.stdout: "pool_max": 2, 2024-09-21T09:43:24.012 INFO:teuthology.orchestra.run.smithi092.stdout: "max_osd": 4, 2024-09-21T09:43:24.012 INFO:teuthology.orchestra.run.smithi092.stdout: "require_min_compat_client": "squid", 2024-09-21T09:43:24.012 INFO:teuthology.orchestra.run.smithi092.stdout: "min_compat_client": "jewel", 2024-09-21T09:43:24.012 INFO:teuthology.orchestra.run.smithi092.stdout: "require_osd_release": "squid", 2024-09-21T09:43:24.012 INFO:teuthology.orchestra.run.smithi092.stdout: "allow_crimson": false, 2024-09-21T09:43:24.013 INFO:teuthology.orchestra.run.smithi092.stdout: "pools": [ 2024-09-21T09:43:24.013 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.013 INFO:teuthology.orchestra.run.smithi092.stdout: "pool": 1, 2024-09-21T09:43:24.013 INFO:teuthology.orchestra.run.smithi092.stdout: "pool_name": ".mgr", 2024-09-21T09:43:24.013 INFO:teuthology.orchestra.run.smithi092.stdout: "create_time": "2024-09-21T09:41:50.040313+0000", 2024-09-21T09:43:24.013 INFO:teuthology.orchestra.run.smithi092.stdout: "flags": 1, 2024-09-21T09:43:24.013 INFO:teuthology.orchestra.run.smithi092.stdout: "flags_names": "hashpspool", 2024-09-21T09:43:24.013 INFO:teuthology.orchestra.run.smithi092.stdout: "type": 1, 2024-09-21T09:43:24.013 INFO:teuthology.orchestra.run.smithi092.stdout: "size": 2, 2024-09-21T09:43:24.013 INFO:teuthology.orchestra.run.smithi092.stdout: "min_size": 1, 2024-09-21T09:43:24.013 INFO:teuthology.orchestra.run.smithi092.stdout: "crush_rule": 0, 2024-09-21T09:43:24.013 INFO:teuthology.orchestra.run.smithi092.stdout: "peering_crush_bucket_count": 0, 2024-09-21T09:43:24.013 INFO:teuthology.orchestra.run.smithi092.stdout: "peering_crush_bucket_target": 0, 2024-09-21T09:43:24.013 INFO:teuthology.orchestra.run.smithi092.stdout: "peering_crush_bucket_barrier": 0, 2024-09-21T09:43:24.013 INFO:teuthology.orchestra.run.smithi092.stdout: "peering_crush_bucket_mandatory_member": 2147483647, 2024-09-21T09:43:24.014 INFO:teuthology.orchestra.run.smithi092.stdout: "is_stretch_pool": false, 2024-09-21T09:43:24.014 INFO:teuthology.orchestra.run.smithi092.stdout: "object_hash": 2, 2024-09-21T09:43:24.014 INFO:teuthology.orchestra.run.smithi092.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:43:24.014 INFO:teuthology.orchestra.run.smithi092.stdout: "pg_num": 1, 2024-09-21T09:43:24.014 INFO:teuthology.orchestra.run.smithi092.stdout: "pg_placement_num": 1, 2024-09-21T09:43:24.014 INFO:teuthology.orchestra.run.smithi092.stdout: "pg_placement_num_target": 1, 2024-09-21T09:43:24.014 INFO:teuthology.orchestra.run.smithi092.stdout: "pg_num_target": 1, 2024-09-21T09:43:24.014 INFO:teuthology.orchestra.run.smithi092.stdout: "pg_num_pending": 1, 2024-09-21T09:43:24.014 INFO:teuthology.orchestra.run.smithi092.stdout: "last_pg_merge_meta": { 2024-09-21T09:43:24.014 INFO:teuthology.orchestra.run.smithi092.stdout: "source_pgid": "0.0", 2024-09-21T09:43:24.014 INFO:teuthology.orchestra.run.smithi092.stdout: "ready_epoch": 0, 2024-09-21T09:43:24.014 INFO:teuthology.orchestra.run.smithi092.stdout: "last_epoch_started": 0, 2024-09-21T09:43:24.014 INFO:teuthology.orchestra.run.smithi092.stdout: "last_epoch_clean": 0, 2024-09-21T09:43:24.014 INFO:teuthology.orchestra.run.smithi092.stdout: "source_version": "0'0", 2024-09-21T09:43:24.015 INFO:teuthology.orchestra.run.smithi092.stdout: "target_version": "0'0" 2024-09-21T09:43:24.015 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.015 INFO:teuthology.orchestra.run.smithi092.stdout: "last_change": "15", 2024-09-21T09:43:24.015 INFO:teuthology.orchestra.run.smithi092.stdout: "last_force_op_resend": "0", 2024-09-21T09:43:24.015 INFO:teuthology.orchestra.run.smithi092.stdout: "last_force_op_resend_prenautilus": "0", 2024-09-21T09:43:24.015 INFO:teuthology.orchestra.run.smithi092.stdout: "last_force_op_resend_preluminous": "0", 2024-09-21T09:43:24.015 INFO:teuthology.orchestra.run.smithi092.stdout: "auid": 0, 2024-09-21T09:43:24.015 INFO:teuthology.orchestra.run.smithi092.stdout: "snap_mode": "selfmanaged", 2024-09-21T09:43:24.015 INFO:teuthology.orchestra.run.smithi092.stdout: "snap_seq": 0, 2024-09-21T09:43:24.015 INFO:teuthology.orchestra.run.smithi092.stdout: "snap_epoch": 0, 2024-09-21T09:43:24.015 INFO:teuthology.orchestra.run.smithi092.stdout: "pool_snaps": [], 2024-09-21T09:43:24.015 INFO:teuthology.orchestra.run.smithi092.stdout: "removed_snaps": "[]", 2024-09-21T09:43:24.015 INFO:teuthology.orchestra.run.smithi092.stdout: "quota_max_bytes": 0, 2024-09-21T09:43:24.015 INFO:teuthology.orchestra.run.smithi092.stdout: "quota_max_objects": 0, 2024-09-21T09:43:24.015 INFO:teuthology.orchestra.run.smithi092.stdout: "tiers": [], 2024-09-21T09:43:24.016 INFO:teuthology.orchestra.run.smithi092.stdout: "tier_of": -1, 2024-09-21T09:43:24.016 INFO:teuthology.orchestra.run.smithi092.stdout: "read_tier": -1, 2024-09-21T09:43:24.016 INFO:teuthology.orchestra.run.smithi092.stdout: "write_tier": -1, 2024-09-21T09:43:24.016 INFO:teuthology.orchestra.run.smithi092.stdout: "cache_mode": "none", 2024-09-21T09:43:24.016 INFO:teuthology.orchestra.run.smithi092.stdout: "target_max_bytes": 0, 2024-09-21T09:43:24.016 INFO:teuthology.orchestra.run.smithi092.stdout: "target_max_objects": 0, 2024-09-21T09:43:24.017 INFO:teuthology.orchestra.run.smithi092.stdout: "cache_target_dirty_ratio_micro": 400000, 2024-09-21T09:43:24.018 INFO:teuthology.orchestra.run.smithi092.stdout: "cache_target_dirty_high_ratio_micro": 600000, 2024-09-21T09:43:24.018 INFO:teuthology.orchestra.run.smithi092.stdout: "cache_target_full_ratio_micro": 800000, 2024-09-21T09:43:24.018 INFO:teuthology.orchestra.run.smithi092.stdout: "cache_min_flush_age": 0, 2024-09-21T09:43:24.018 INFO:teuthology.orchestra.run.smithi092.stdout: "cache_min_evict_age": 0, 2024-09-21T09:43:24.018 INFO:teuthology.orchestra.run.smithi092.stdout: "erasure_code_profile": "", 2024-09-21T09:43:24.018 INFO:teuthology.orchestra.run.smithi092.stdout: "hit_set_params": { 2024-09-21T09:43:24.018 INFO:teuthology.orchestra.run.smithi092.stdout: "type": "none" 2024-09-21T09:43:24.018 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.018 INFO:teuthology.orchestra.run.smithi092.stdout: "hit_set_period": 0, 2024-09-21T09:43:24.018 INFO:teuthology.orchestra.run.smithi092.stdout: "hit_set_count": 0, 2024-09-21T09:43:24.018 INFO:teuthology.orchestra.run.smithi092.stdout: "use_gmt_hitset": true, 2024-09-21T09:43:24.018 INFO:teuthology.orchestra.run.smithi092.stdout: "min_read_recency_for_promote": 0, 2024-09-21T09:43:24.018 INFO:teuthology.orchestra.run.smithi092.stdout: "min_write_recency_for_promote": 0, 2024-09-21T09:43:24.018 INFO:teuthology.orchestra.run.smithi092.stdout: "hit_set_grade_decay_rate": 0, 2024-09-21T09:43:24.019 INFO:teuthology.orchestra.run.smithi092.stdout: "hit_set_search_last_n": 0, 2024-09-21T09:43:24.019 INFO:teuthology.orchestra.run.smithi092.stdout: "grade_table": [], 2024-09-21T09:43:24.019 INFO:teuthology.orchestra.run.smithi092.stdout: "stripe_width": 0, 2024-09-21T09:43:24.019 INFO:teuthology.orchestra.run.smithi092.stdout: "expected_num_objects": 0, 2024-09-21T09:43:24.019 INFO:teuthology.orchestra.run.smithi092.stdout: "fast_read": false, 2024-09-21T09:43:24.019 INFO:teuthology.orchestra.run.smithi092.stdout: "options": { 2024-09-21T09:43:24.019 INFO:teuthology.orchestra.run.smithi092.stdout: "pg_num_max": 32, 2024-09-21T09:43:24.019 INFO:teuthology.orchestra.run.smithi092.stdout: "pg_num_min": 1 2024-09-21T09:43:24.019 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.019 INFO:teuthology.orchestra.run.smithi092.stdout: "application_metadata": { 2024-09-21T09:43:24.019 INFO:teuthology.orchestra.run.smithi092.stdout: "mgr": {} 2024-09-21T09:43:24.019 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.019 INFO:teuthology.orchestra.run.smithi092.stdout: "read_balance": { 2024-09-21T09:43:24.019 INFO:teuthology.orchestra.run.smithi092.stdout: "score_type": "Fair distribution", 2024-09-21T09:43:24.020 INFO:teuthology.orchestra.run.smithi092.stdout: "score_acting": 4, 2024-09-21T09:43:24.020 INFO:teuthology.orchestra.run.smithi092.stdout: "score_stable": 4, 2024-09-21T09:43:24.020 INFO:teuthology.orchestra.run.smithi092.stdout: "optimal_score": 0.5, 2024-09-21T09:43:24.020 INFO:teuthology.orchestra.run.smithi092.stdout: "raw_score_acting": 2, 2024-09-21T09:43:24.020 INFO:teuthology.orchestra.run.smithi092.stdout: "raw_score_stable": 2, 2024-09-21T09:43:24.020 INFO:teuthology.orchestra.run.smithi092.stdout: "primary_affinity_weighted": 1, 2024-09-21T09:43:24.020 INFO:teuthology.orchestra.run.smithi092.stdout: "average_primary_affinity": 1, 2024-09-21T09:43:24.020 INFO:teuthology.orchestra.run.smithi092.stdout: "average_primary_affinity_weighted": 1 2024-09-21T09:43:24.020 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.020 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.020 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.020 INFO:teuthology.orchestra.run.smithi092.stdout: "pool": 2, 2024-09-21T09:43:24.020 INFO:teuthology.orchestra.run.smithi092.stdout: "pool_name": "rbd", 2024-09-21T09:43:24.020 INFO:teuthology.orchestra.run.smithi092.stdout: "create_time": "2024-09-21T09:41:52.054798+0000", 2024-09-21T09:43:24.020 INFO:teuthology.orchestra.run.smithi092.stdout: "flags": 8193, 2024-09-21T09:43:24.021 INFO:teuthology.orchestra.run.smithi092.stdout: "flags_names": "hashpspool,selfmanaged_snaps", 2024-09-21T09:43:24.021 INFO:teuthology.orchestra.run.smithi092.stdout: "type": 1, 2024-09-21T09:43:24.021 INFO:teuthology.orchestra.run.smithi092.stdout: "size": 2, 2024-09-21T09:43:24.021 INFO:teuthology.orchestra.run.smithi092.stdout: "min_size": 1, 2024-09-21T09:43:24.021 INFO:teuthology.orchestra.run.smithi092.stdout: "crush_rule": 0, 2024-09-21T09:43:24.021 INFO:teuthology.orchestra.run.smithi092.stdout: "peering_crush_bucket_count": 0, 2024-09-21T09:43:24.021 INFO:teuthology.orchestra.run.smithi092.stdout: "peering_crush_bucket_target": 0, 2024-09-21T09:43:24.021 INFO:teuthology.orchestra.run.smithi092.stdout: "peering_crush_bucket_barrier": 0, 2024-09-21T09:43:24.021 INFO:teuthology.orchestra.run.smithi092.stdout: "peering_crush_bucket_mandatory_member": 2147483647, 2024-09-21T09:43:24.021 INFO:teuthology.orchestra.run.smithi092.stdout: "is_stretch_pool": false, 2024-09-21T09:43:24.021 INFO:teuthology.orchestra.run.smithi092.stdout: "object_hash": 2, 2024-09-21T09:43:24.021 INFO:teuthology.orchestra.run.smithi092.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:43:24.021 INFO:teuthology.orchestra.run.smithi092.stdout: "pg_num": 8, 2024-09-21T09:43:24.021 INFO:teuthology.orchestra.run.smithi092.stdout: "pg_placement_num": 8, 2024-09-21T09:43:24.021 INFO:teuthology.orchestra.run.smithi092.stdout: "pg_placement_num_target": 8, 2024-09-21T09:43:24.022 INFO:teuthology.orchestra.run.smithi092.stdout: "pg_num_target": 8, 2024-09-21T09:43:24.022 INFO:teuthology.orchestra.run.smithi092.stdout: "pg_num_pending": 8, 2024-09-21T09:43:24.022 INFO:teuthology.orchestra.run.smithi092.stdout: "last_pg_merge_meta": { 2024-09-21T09:43:24.022 INFO:teuthology.orchestra.run.smithi092.stdout: "source_pgid": "0.0", 2024-09-21T09:43:24.022 INFO:teuthology.orchestra.run.smithi092.stdout: "ready_epoch": 0, 2024-09-21T09:43:24.022 INFO:teuthology.orchestra.run.smithi092.stdout: "last_epoch_started": 0, 2024-09-21T09:43:24.022 INFO:teuthology.orchestra.run.smithi092.stdout: "last_epoch_clean": 0, 2024-09-21T09:43:24.022 INFO:teuthology.orchestra.run.smithi092.stdout: "source_version": "0'0", 2024-09-21T09:43:24.022 INFO:teuthology.orchestra.run.smithi092.stdout: "target_version": "0'0" 2024-09-21T09:43:24.022 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.022 INFO:teuthology.orchestra.run.smithi092.stdout: "last_change": "18", 2024-09-21T09:43:24.022 INFO:teuthology.orchestra.run.smithi092.stdout: "last_force_op_resend": "0", 2024-09-21T09:43:24.022 INFO:teuthology.orchestra.run.smithi092.stdout: "last_force_op_resend_prenautilus": "0", 2024-09-21T09:43:24.023 INFO:teuthology.orchestra.run.smithi092.stdout: "last_force_op_resend_preluminous": "0", 2024-09-21T09:43:24.023 INFO:teuthology.orchestra.run.smithi092.stdout: "auid": 0, 2024-09-21T09:43:24.023 INFO:teuthology.orchestra.run.smithi092.stdout: "snap_mode": "selfmanaged", 2024-09-21T09:43:24.023 INFO:teuthology.orchestra.run.smithi092.stdout: "snap_seq": 2, 2024-09-21T09:43:24.023 INFO:teuthology.orchestra.run.smithi092.stdout: "snap_epoch": 18, 2024-09-21T09:43:24.023 INFO:teuthology.orchestra.run.smithi092.stdout: "pool_snaps": [], 2024-09-21T09:43:24.023 INFO:teuthology.orchestra.run.smithi092.stdout: "removed_snaps": "[]", 2024-09-21T09:43:24.023 INFO:teuthology.orchestra.run.smithi092.stdout: "quota_max_bytes": 0, 2024-09-21T09:43:24.023 INFO:teuthology.orchestra.run.smithi092.stdout: "quota_max_objects": 0, 2024-09-21T09:43:24.023 INFO:teuthology.orchestra.run.smithi092.stdout: "tiers": [], 2024-09-21T09:43:24.023 INFO:teuthology.orchestra.run.smithi092.stdout: "tier_of": -1, 2024-09-21T09:43:24.023 INFO:teuthology.orchestra.run.smithi092.stdout: "read_tier": -1, 2024-09-21T09:43:24.023 INFO:teuthology.orchestra.run.smithi092.stdout: "write_tier": -1, 2024-09-21T09:43:24.023 INFO:teuthology.orchestra.run.smithi092.stdout: "cache_mode": "none", 2024-09-21T09:43:24.024 INFO:teuthology.orchestra.run.smithi092.stdout: "target_max_bytes": 0, 2024-09-21T09:43:24.024 INFO:teuthology.orchestra.run.smithi092.stdout: "target_max_objects": 0, 2024-09-21T09:43:24.024 INFO:teuthology.orchestra.run.smithi092.stdout: "cache_target_dirty_ratio_micro": 400000, 2024-09-21T09:43:24.024 INFO:teuthology.orchestra.run.smithi092.stdout: "cache_target_dirty_high_ratio_micro": 600000, 2024-09-21T09:43:24.024 INFO:teuthology.orchestra.run.smithi092.stdout: "cache_target_full_ratio_micro": 800000, 2024-09-21T09:43:24.024 INFO:teuthology.orchestra.run.smithi092.stdout: "cache_min_flush_age": 0, 2024-09-21T09:43:24.024 INFO:teuthology.orchestra.run.smithi092.stdout: "cache_min_evict_age": 0, 2024-09-21T09:43:24.024 INFO:teuthology.orchestra.run.smithi092.stdout: "erasure_code_profile": "", 2024-09-21T09:43:24.024 INFO:teuthology.orchestra.run.smithi092.stdout: "hit_set_params": { 2024-09-21T09:43:24.024 INFO:teuthology.orchestra.run.smithi092.stdout: "type": "none" 2024-09-21T09:43:24.024 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.024 INFO:teuthology.orchestra.run.smithi092.stdout: "hit_set_period": 0, 2024-09-21T09:43:24.024 INFO:teuthology.orchestra.run.smithi092.stdout: "hit_set_count": 0, 2024-09-21T09:43:24.024 INFO:teuthology.orchestra.run.smithi092.stdout: "use_gmt_hitset": true, 2024-09-21T09:43:24.024 INFO:teuthology.orchestra.run.smithi092.stdout: "min_read_recency_for_promote": 0, 2024-09-21T09:43:24.025 INFO:teuthology.orchestra.run.smithi092.stdout: "min_write_recency_for_promote": 0, 2024-09-21T09:43:24.025 INFO:teuthology.orchestra.run.smithi092.stdout: "hit_set_grade_decay_rate": 0, 2024-09-21T09:43:24.025 INFO:teuthology.orchestra.run.smithi092.stdout: "hit_set_search_last_n": 0, 2024-09-21T09:43:24.025 INFO:teuthology.orchestra.run.smithi092.stdout: "grade_table": [], 2024-09-21T09:43:24.025 INFO:teuthology.orchestra.run.smithi092.stdout: "stripe_width": 0, 2024-09-21T09:43:24.025 INFO:teuthology.orchestra.run.smithi092.stdout: "expected_num_objects": 0, 2024-09-21T09:43:24.025 INFO:teuthology.orchestra.run.smithi092.stdout: "fast_read": false, 2024-09-21T09:43:24.025 INFO:teuthology.orchestra.run.smithi092.stdout: "options": {}, 2024-09-21T09:43:24.025 INFO:teuthology.orchestra.run.smithi092.stdout: "application_metadata": { 2024-09-21T09:43:24.025 INFO:teuthology.orchestra.run.smithi092.stdout: "rbd": {} 2024-09-21T09:43:24.025 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.025 INFO:teuthology.orchestra.run.smithi092.stdout: "read_balance": { 2024-09-21T09:43:24.025 INFO:teuthology.orchestra.run.smithi092.stdout: "score_type": "Fair distribution", 2024-09-21T09:43:24.026 INFO:teuthology.orchestra.run.smithi092.stdout: "score_acting": 2.5, 2024-09-21T09:43:24.026 INFO:teuthology.orchestra.run.smithi092.stdout: "score_stable": 2.5, 2024-09-21T09:43:24.026 INFO:teuthology.orchestra.run.smithi092.stdout: "optimal_score": 1, 2024-09-21T09:43:24.026 INFO:teuthology.orchestra.run.smithi092.stdout: "raw_score_acting": 2.5, 2024-09-21T09:43:24.026 INFO:teuthology.orchestra.run.smithi092.stdout: "raw_score_stable": 2.5, 2024-09-21T09:43:24.026 INFO:teuthology.orchestra.run.smithi092.stdout: "primary_affinity_weighted": 1, 2024-09-21T09:43:24.026 INFO:teuthology.orchestra.run.smithi092.stdout: "average_primary_affinity": 1, 2024-09-21T09:43:24.026 INFO:teuthology.orchestra.run.smithi092.stdout: "average_primary_affinity_weighted": 1 2024-09-21T09:43:24.026 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.026 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.026 INFO:teuthology.orchestra.run.smithi092.stdout: ], 2024-09-21T09:43:24.026 INFO:teuthology.orchestra.run.smithi092.stdout: "osds": [ 2024-09-21T09:43:24.026 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.026 INFO:teuthology.orchestra.run.smithi092.stdout: "osd": 0, 2024-09-21T09:43:24.026 INFO:teuthology.orchestra.run.smithi092.stdout: "uuid": "f3314f59-3374-4072-85c9-f8c4e458efc8", 2024-09-21T09:43:24.027 INFO:teuthology.orchestra.run.smithi092.stdout: "up": 1, 2024-09-21T09:43:24.027 INFO:teuthology.orchestra.run.smithi092.stdout: "in": 1, 2024-09-21T09:43:24.027 INFO:teuthology.orchestra.run.smithi092.stdout: "weight": 1, 2024-09-21T09:43:24.027 INFO:teuthology.orchestra.run.smithi092.stdout: "primary_affinity": 1, 2024-09-21T09:43:24.027 INFO:teuthology.orchestra.run.smithi092.stdout: "last_clean_begin": 11, 2024-09-21T09:43:24.027 INFO:teuthology.orchestra.run.smithi092.stdout: "last_clean_end": 21, 2024-09-21T09:43:24.027 INFO:teuthology.orchestra.run.smithi092.stdout: "up_from": 27, 2024-09-21T09:43:24.027 INFO:teuthology.orchestra.run.smithi092.stdout: "up_thru": 27, 2024-09-21T09:43:24.027 INFO:teuthology.orchestra.run.smithi092.stdout: "down_at": 22, 2024-09-21T09:43:24.027 INFO:teuthology.orchestra.run.smithi092.stdout: "lost_at": 0, 2024-09-21T09:43:24.027 INFO:teuthology.orchestra.run.smithi092.stdout: "public_addrs": { 2024-09-21T09:43:24.027 INFO:teuthology.orchestra.run.smithi092.stdout: "addrvec": [ 2024-09-21T09:43:24.027 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.027 INFO:teuthology.orchestra.run.smithi092.stdout: "type": "v1", 2024-09-21T09:43:24.028 INFO:teuthology.orchestra.run.smithi092.stdout: "addr": "172.21.15.92:6805", 2024-09-21T09:43:24.028 INFO:teuthology.orchestra.run.smithi092.stdout: "nonce": 7087154 2024-09-21T09:43:24.028 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.028 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2024-09-21T09:43:24.028 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.028 INFO:teuthology.orchestra.run.smithi092.stdout: "cluster_addrs": { 2024-09-21T09:43:24.028 INFO:teuthology.orchestra.run.smithi092.stdout: "addrvec": [ 2024-09-21T09:43:24.028 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.028 INFO:teuthology.orchestra.run.smithi092.stdout: "type": "v1", 2024-09-21T09:43:24.028 INFO:teuthology.orchestra.run.smithi092.stdout: "addr": "172.21.15.92:6806", 2024-09-21T09:43:24.028 INFO:teuthology.orchestra.run.smithi092.stdout: "nonce": 7087154 2024-09-21T09:43:24.028 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.028 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2024-09-21T09:43:24.028 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.028 INFO:teuthology.orchestra.run.smithi092.stdout: "heartbeat_back_addrs": { 2024-09-21T09:43:24.029 INFO:teuthology.orchestra.run.smithi092.stdout: "addrvec": [ 2024-09-21T09:43:24.029 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.029 INFO:teuthology.orchestra.run.smithi092.stdout: "type": "v1", 2024-09-21T09:43:24.029 INFO:teuthology.orchestra.run.smithi092.stdout: "addr": "172.21.15.92:6808", 2024-09-21T09:43:24.029 INFO:teuthology.orchestra.run.smithi092.stdout: "nonce": 7087154 2024-09-21T09:43:24.029 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.029 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2024-09-21T09:43:24.029 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.029 INFO:teuthology.orchestra.run.smithi092.stdout: "heartbeat_front_addrs": { 2024-09-21T09:43:24.029 INFO:teuthology.orchestra.run.smithi092.stdout: "addrvec": [ 2024-09-21T09:43:24.029 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.029 INFO:teuthology.orchestra.run.smithi092.stdout: "type": "v1", 2024-09-21T09:43:24.029 INFO:teuthology.orchestra.run.smithi092.stdout: "addr": "172.21.15.92:6807", 2024-09-21T09:43:24.029 INFO:teuthology.orchestra.run.smithi092.stdout: "nonce": 7087154 2024-09-21T09:43:24.030 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.030 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2024-09-21T09:43:24.030 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.030 INFO:teuthology.orchestra.run.smithi092.stdout: "public_addr": "172.21.15.92:6805/7087154", 2024-09-21T09:43:24.030 INFO:teuthology.orchestra.run.smithi092.stdout: "cluster_addr": "172.21.15.92:6806/7087154", 2024-09-21T09:43:24.030 INFO:teuthology.orchestra.run.smithi092.stdout: "heartbeat_back_addr": "172.21.15.92:6808/7087154", 2024-09-21T09:43:24.030 INFO:teuthology.orchestra.run.smithi092.stdout: "heartbeat_front_addr": "172.21.15.92:6807/7087154", 2024-09-21T09:43:24.030 INFO:teuthology.orchestra.run.smithi092.stdout: "state": [ 2024-09-21T09:43:24.030 INFO:teuthology.orchestra.run.smithi092.stdout: "exists", 2024-09-21T09:43:24.030 INFO:teuthology.orchestra.run.smithi092.stdout: "up" 2024-09-21T09:43:24.030 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2024-09-21T09:43:24.030 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.030 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.030 INFO:teuthology.orchestra.run.smithi092.stdout: "osd": 1, 2024-09-21T09:43:24.031 INFO:teuthology.orchestra.run.smithi092.stdout: "uuid": "cf0486af-ba68-4763-a364-c1f2181604a7", 2024-09-21T09:43:24.031 INFO:teuthology.orchestra.run.smithi092.stdout: "up": 1, 2024-09-21T09:43:24.031 INFO:teuthology.orchestra.run.smithi092.stdout: "in": 1, 2024-09-21T09:43:24.031 INFO:teuthology.orchestra.run.smithi092.stdout: "weight": 1, 2024-09-21T09:43:24.031 INFO:teuthology.orchestra.run.smithi092.stdout: "primary_affinity": 1, 2024-09-21T09:43:24.031 INFO:teuthology.orchestra.run.smithi092.stdout: "last_clean_begin": 11, 2024-09-21T09:43:24.031 INFO:teuthology.orchestra.run.smithi092.stdout: "last_clean_end": 21, 2024-09-21T09:43:24.031 INFO:teuthology.orchestra.run.smithi092.stdout: "up_from": 28, 2024-09-21T09:43:24.031 INFO:teuthology.orchestra.run.smithi092.stdout: "up_thru": 28, 2024-09-21T09:43:24.031 INFO:teuthology.orchestra.run.smithi092.stdout: "down_at": 23, 2024-09-21T09:43:24.031 INFO:teuthology.orchestra.run.smithi092.stdout: "lost_at": 0, 2024-09-21T09:43:24.031 INFO:teuthology.orchestra.run.smithi092.stdout: "public_addrs": { 2024-09-21T09:43:24.031 INFO:teuthology.orchestra.run.smithi092.stdout: "addrvec": [ 2024-09-21T09:43:24.031 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.031 INFO:teuthology.orchestra.run.smithi092.stdout: "type": "v1", 2024-09-21T09:43:24.032 INFO:teuthology.orchestra.run.smithi092.stdout: "addr": "172.21.15.92:6801", 2024-09-21T09:43:24.032 INFO:teuthology.orchestra.run.smithi092.stdout: "nonce": 2577383826 2024-09-21T09:43:24.032 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.032 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2024-09-21T09:43:24.032 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.032 INFO:teuthology.orchestra.run.smithi092.stdout: "cluster_addrs": { 2024-09-21T09:43:24.032 INFO:teuthology.orchestra.run.smithi092.stdout: "addrvec": [ 2024-09-21T09:43:24.032 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.032 INFO:teuthology.orchestra.run.smithi092.stdout: "type": "v1", 2024-09-21T09:43:24.032 INFO:teuthology.orchestra.run.smithi092.stdout: "addr": "172.21.15.92:6802", 2024-09-21T09:43:24.032 INFO:teuthology.orchestra.run.smithi092.stdout: "nonce": 2577383826 2024-09-21T09:43:24.032 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.032 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2024-09-21T09:43:24.032 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.032 INFO:teuthology.orchestra.run.smithi092.stdout: "heartbeat_back_addrs": { 2024-09-21T09:43:24.033 INFO:teuthology.orchestra.run.smithi092.stdout: "addrvec": [ 2024-09-21T09:43:24.033 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.033 INFO:teuthology.orchestra.run.smithi092.stdout: "type": "v1", 2024-09-21T09:43:24.033 INFO:teuthology.orchestra.run.smithi092.stdout: "addr": "172.21.15.92:6804", 2024-09-21T09:43:24.033 INFO:teuthology.orchestra.run.smithi092.stdout: "nonce": 2577383826 2024-09-21T09:43:24.033 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.033 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2024-09-21T09:43:24.033 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.033 INFO:teuthology.orchestra.run.smithi092.stdout: "heartbeat_front_addrs": { 2024-09-21T09:43:24.033 INFO:teuthology.orchestra.run.smithi092.stdout: "addrvec": [ 2024-09-21T09:43:24.033 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.033 INFO:teuthology.orchestra.run.smithi092.stdout: "type": "v1", 2024-09-21T09:43:24.033 INFO:teuthology.orchestra.run.smithi092.stdout: "addr": "172.21.15.92:6803", 2024-09-21T09:43:24.033 INFO:teuthology.orchestra.run.smithi092.stdout: "nonce": 2577383826 2024-09-21T09:43:24.034 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.034 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2024-09-21T09:43:24.034 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.034 INFO:teuthology.orchestra.run.smithi092.stdout: "public_addr": "172.21.15.92:6801/2577383826", 2024-09-21T09:43:24.034 INFO:teuthology.orchestra.run.smithi092.stdout: "cluster_addr": "172.21.15.92:6802/2577383826", 2024-09-21T09:43:24.034 INFO:teuthology.orchestra.run.smithi092.stdout: "heartbeat_back_addr": "172.21.15.92:6804/2577383826", 2024-09-21T09:43:24.034 INFO:teuthology.orchestra.run.smithi092.stdout: "heartbeat_front_addr": "172.21.15.92:6803/2577383826", 2024-09-21T09:43:24.034 INFO:teuthology.orchestra.run.smithi092.stdout: "state": [ 2024-09-21T09:43:24.034 INFO:teuthology.orchestra.run.smithi092.stdout: "exists", 2024-09-21T09:43:24.034 INFO:teuthology.orchestra.run.smithi092.stdout: "up" 2024-09-21T09:43:24.034 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2024-09-21T09:43:24.034 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.034 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.034 INFO:teuthology.orchestra.run.smithi092.stdout: "osd": 2, 2024-09-21T09:43:24.035 INFO:teuthology.orchestra.run.smithi092.stdout: "uuid": "5c587421-3409-4469-a2ee-b2ea20cd45e5", 2024-09-21T09:43:24.035 INFO:teuthology.orchestra.run.smithi092.stdout: "up": 1, 2024-09-21T09:43:24.035 INFO:teuthology.orchestra.run.smithi092.stdout: "in": 1, 2024-09-21T09:43:24.035 INFO:teuthology.orchestra.run.smithi092.stdout: "weight": 1, 2024-09-21T09:43:24.035 INFO:teuthology.orchestra.run.smithi092.stdout: "primary_affinity": 1, 2024-09-21T09:43:24.035 INFO:teuthology.orchestra.run.smithi092.stdout: "last_clean_begin": 11, 2024-09-21T09:43:24.035 INFO:teuthology.orchestra.run.smithi092.stdout: "last_clean_end": 23, 2024-09-21T09:43:24.035 INFO:teuthology.orchestra.run.smithi092.stdout: "up_from": 29, 2024-09-21T09:43:24.035 INFO:teuthology.orchestra.run.smithi092.stdout: "up_thru": 30, 2024-09-21T09:43:24.035 INFO:teuthology.orchestra.run.smithi092.stdout: "down_at": 24, 2024-09-21T09:43:24.035 INFO:teuthology.orchestra.run.smithi092.stdout: "lost_at": 0, 2024-09-21T09:43:24.035 INFO:teuthology.orchestra.run.smithi092.stdout: "public_addrs": { 2024-09-21T09:43:24.035 INFO:teuthology.orchestra.run.smithi092.stdout: "addrvec": [ 2024-09-21T09:43:24.035 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.036 INFO:teuthology.orchestra.run.smithi092.stdout: "type": "v1", 2024-09-21T09:43:24.036 INFO:teuthology.orchestra.run.smithi092.stdout: "addr": "172.21.15.92:6809", 2024-09-21T09:43:24.036 INFO:teuthology.orchestra.run.smithi092.stdout: "nonce": 4235634095 2024-09-21T09:43:24.036 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.036 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2024-09-21T09:43:24.036 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.036 INFO:teuthology.orchestra.run.smithi092.stdout: "cluster_addrs": { 2024-09-21T09:43:24.036 INFO:teuthology.orchestra.run.smithi092.stdout: "addrvec": [ 2024-09-21T09:43:24.036 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.036 INFO:teuthology.orchestra.run.smithi092.stdout: "type": "v1", 2024-09-21T09:43:24.036 INFO:teuthology.orchestra.run.smithi092.stdout: "addr": "172.21.15.92:6810", 2024-09-21T09:43:24.036 INFO:teuthology.orchestra.run.smithi092.stdout: "nonce": 4235634095 2024-09-21T09:43:24.036 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.036 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2024-09-21T09:43:24.037 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.037 INFO:teuthology.orchestra.run.smithi092.stdout: "heartbeat_back_addrs": { 2024-09-21T09:43:24.037 INFO:teuthology.orchestra.run.smithi092.stdout: "addrvec": [ 2024-09-21T09:43:24.037 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.037 INFO:teuthology.orchestra.run.smithi092.stdout: "type": "v1", 2024-09-21T09:43:24.037 INFO:teuthology.orchestra.run.smithi092.stdout: "addr": "172.21.15.92:6812", 2024-09-21T09:43:24.037 INFO:teuthology.orchestra.run.smithi092.stdout: "nonce": 4235634095 2024-09-21T09:43:24.037 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.037 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2024-09-21T09:43:24.037 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.037 INFO:teuthology.orchestra.run.smithi092.stdout: "heartbeat_front_addrs": { 2024-09-21T09:43:24.037 INFO:teuthology.orchestra.run.smithi092.stdout: "addrvec": [ 2024-09-21T09:43:24.037 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.037 INFO:teuthology.orchestra.run.smithi092.stdout: "type": "v1", 2024-09-21T09:43:24.038 INFO:teuthology.orchestra.run.smithi092.stdout: "addr": "172.21.15.92:6811", 2024-09-21T09:43:24.038 INFO:teuthology.orchestra.run.smithi092.stdout: "nonce": 4235634095 2024-09-21T09:43:24.038 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.038 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2024-09-21T09:43:24.038 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.038 INFO:teuthology.orchestra.run.smithi092.stdout: "public_addr": "172.21.15.92:6809/4235634095", 2024-09-21T09:43:24.038 INFO:teuthology.orchestra.run.smithi092.stdout: "cluster_addr": "172.21.15.92:6810/4235634095", 2024-09-21T09:43:24.038 INFO:teuthology.orchestra.run.smithi092.stdout: "heartbeat_back_addr": "172.21.15.92:6812/4235634095", 2024-09-21T09:43:24.038 INFO:teuthology.orchestra.run.smithi092.stdout: "heartbeat_front_addr": "172.21.15.92:6811/4235634095", 2024-09-21T09:43:24.038 INFO:teuthology.orchestra.run.smithi092.stdout: "state": [ 2024-09-21T09:43:24.038 INFO:teuthology.orchestra.run.smithi092.stdout: "exists", 2024-09-21T09:43:24.038 INFO:teuthology.orchestra.run.smithi092.stdout: "up" 2024-09-21T09:43:24.038 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2024-09-21T09:43:24.038 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.038 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.039 INFO:teuthology.orchestra.run.smithi092.stdout: "osd": 3, 2024-09-21T09:43:24.039 INFO:teuthology.orchestra.run.smithi092.stdout: "uuid": "b98663b7-de4d-4c12-ac91-8c33f159da35", 2024-09-21T09:43:24.039 INFO:teuthology.orchestra.run.smithi092.stdout: "up": 1, 2024-09-21T09:43:24.039 INFO:teuthology.orchestra.run.smithi092.stdout: "in": 1, 2024-09-21T09:43:24.039 INFO:teuthology.orchestra.run.smithi092.stdout: "weight": 1, 2024-09-21T09:43:24.039 INFO:teuthology.orchestra.run.smithi092.stdout: "primary_affinity": 1, 2024-09-21T09:43:24.039 INFO:teuthology.orchestra.run.smithi092.stdout: "last_clean_begin": 9, 2024-09-21T09:43:24.039 INFO:teuthology.orchestra.run.smithi092.stdout: "last_clean_end": 24, 2024-09-21T09:43:24.039 INFO:teuthology.orchestra.run.smithi092.stdout: "up_from": 30, 2024-09-21T09:43:24.039 INFO:teuthology.orchestra.run.smithi092.stdout: "up_thru": 30, 2024-09-21T09:43:24.039 INFO:teuthology.orchestra.run.smithi092.stdout: "down_at": 25, 2024-09-21T09:43:24.039 INFO:teuthology.orchestra.run.smithi092.stdout: "lost_at": 0, 2024-09-21T09:43:24.039 INFO:teuthology.orchestra.run.smithi092.stdout: "public_addrs": { 2024-09-21T09:43:24.039 INFO:teuthology.orchestra.run.smithi092.stdout: "addrvec": [ 2024-09-21T09:43:24.039 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.040 INFO:teuthology.orchestra.run.smithi092.stdout: "type": "v1", 2024-09-21T09:43:24.040 INFO:teuthology.orchestra.run.smithi092.stdout: "addr": "172.21.15.92:6813", 2024-09-21T09:43:24.040 INFO:teuthology.orchestra.run.smithi092.stdout: "nonce": 138232475 2024-09-21T09:43:24.040 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.040 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2024-09-21T09:43:24.040 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.040 INFO:teuthology.orchestra.run.smithi092.stdout: "cluster_addrs": { 2024-09-21T09:43:24.040 INFO:teuthology.orchestra.run.smithi092.stdout: "addrvec": [ 2024-09-21T09:43:24.040 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.040 INFO:teuthology.orchestra.run.smithi092.stdout: "type": "v1", 2024-09-21T09:43:24.040 INFO:teuthology.orchestra.run.smithi092.stdout: "addr": "172.21.15.92:6814", 2024-09-21T09:43:24.040 INFO:teuthology.orchestra.run.smithi092.stdout: "nonce": 138232475 2024-09-21T09:43:24.040 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.040 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2024-09-21T09:43:24.040 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.041 INFO:teuthology.orchestra.run.smithi092.stdout: "heartbeat_back_addrs": { 2024-09-21T09:43:24.041 INFO:teuthology.orchestra.run.smithi092.stdout: "addrvec": [ 2024-09-21T09:43:24.041 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.041 INFO:teuthology.orchestra.run.smithi092.stdout: "type": "v1", 2024-09-21T09:43:24.041 INFO:teuthology.orchestra.run.smithi092.stdout: "addr": "172.21.15.92:6816", 2024-09-21T09:43:24.041 INFO:teuthology.orchestra.run.smithi092.stdout: "nonce": 138232475 2024-09-21T09:43:24.041 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.041 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2024-09-21T09:43:24.041 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.041 INFO:teuthology.orchestra.run.smithi092.stdout: "heartbeat_front_addrs": { 2024-09-21T09:43:24.041 INFO:teuthology.orchestra.run.smithi092.stdout: "addrvec": [ 2024-09-21T09:43:24.041 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.041 INFO:teuthology.orchestra.run.smithi092.stdout: "type": "v1", 2024-09-21T09:43:24.041 INFO:teuthology.orchestra.run.smithi092.stdout: "addr": "172.21.15.92:6815", 2024-09-21T09:43:24.041 INFO:teuthology.orchestra.run.smithi092.stdout: "nonce": 138232475 2024-09-21T09:43:24.042 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.042 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2024-09-21T09:43:24.042 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.042 INFO:teuthology.orchestra.run.smithi092.stdout: "public_addr": "172.21.15.92:6813/138232475", 2024-09-21T09:43:24.042 INFO:teuthology.orchestra.run.smithi092.stdout: "cluster_addr": "172.21.15.92:6814/138232475", 2024-09-21T09:43:24.042 INFO:teuthology.orchestra.run.smithi092.stdout: "heartbeat_back_addr": "172.21.15.92:6816/138232475", 2024-09-21T09:43:24.042 INFO:teuthology.orchestra.run.smithi092.stdout: "heartbeat_front_addr": "172.21.15.92:6815/138232475", 2024-09-21T09:43:24.042 INFO:teuthology.orchestra.run.smithi092.stdout: "state": [ 2024-09-21T09:43:24.042 INFO:teuthology.orchestra.run.smithi092.stdout: "exists", 2024-09-21T09:43:24.042 INFO:teuthology.orchestra.run.smithi092.stdout: "up" 2024-09-21T09:43:24.042 INFO:teuthology.orchestra.run.smithi092.stdout: ] 2024-09-21T09:43:24.042 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.042 INFO:teuthology.orchestra.run.smithi092.stdout: ], 2024-09-21T09:43:24.042 INFO:teuthology.orchestra.run.smithi092.stdout: "osd_xinfo": [ 2024-09-21T09:43:24.042 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.043 INFO:teuthology.orchestra.run.smithi092.stdout: "osd": 0, 2024-09-21T09:43:24.043 INFO:teuthology.orchestra.run.smithi092.stdout: "down_stamp": "0.000000", 2024-09-21T09:43:24.043 INFO:teuthology.orchestra.run.smithi092.stdout: "laggy_probability": 0, 2024-09-21T09:43:24.043 INFO:teuthology.orchestra.run.smithi092.stdout: "laggy_interval": 0, 2024-09-21T09:43:24.043 INFO:teuthology.orchestra.run.smithi092.stdout: "features": 4540719139924082687, 2024-09-21T09:43:24.043 INFO:teuthology.orchestra.run.smithi092.stdout: "old_weight": 0, 2024-09-21T09:43:24.043 INFO:teuthology.orchestra.run.smithi092.stdout: "last_purged_snaps_scrub": "2024-09-21T09:41:44.934317+0000", 2024-09-21T09:43:24.043 INFO:teuthology.orchestra.run.smithi092.stdout: "dead_epoch": 22 2024-09-21T09:43:24.043 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.043 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.043 INFO:teuthology.orchestra.run.smithi092.stdout: "osd": 1, 2024-09-21T09:43:24.043 INFO:teuthology.orchestra.run.smithi092.stdout: "down_stamp": "0.000000", 2024-09-21T09:43:24.043 INFO:teuthology.orchestra.run.smithi092.stdout: "laggy_probability": 0, 2024-09-21T09:43:24.043 INFO:teuthology.orchestra.run.smithi092.stdout: "laggy_interval": 0, 2024-09-21T09:43:24.043 INFO:teuthology.orchestra.run.smithi092.stdout: "features": 4540719139924082687, 2024-09-21T09:43:24.044 INFO:teuthology.orchestra.run.smithi092.stdout: "old_weight": 0, 2024-09-21T09:43:24.044 INFO:teuthology.orchestra.run.smithi092.stdout: "last_purged_snaps_scrub": "2024-09-21T09:41:45.144020+0000", 2024-09-21T09:43:24.044 INFO:teuthology.orchestra.run.smithi092.stdout: "dead_epoch": 23 2024-09-21T09:43:24.044 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.044 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.044 INFO:teuthology.orchestra.run.smithi092.stdout: "osd": 2, 2024-09-21T09:43:24.044 INFO:teuthology.orchestra.run.smithi092.stdout: "down_stamp": "0.000000", 2024-09-21T09:43:24.044 INFO:teuthology.orchestra.run.smithi092.stdout: "laggy_probability": 0, 2024-09-21T09:43:24.044 INFO:teuthology.orchestra.run.smithi092.stdout: "laggy_interval": 0, 2024-09-21T09:43:24.044 INFO:teuthology.orchestra.run.smithi092.stdout: "features": 4540719139924082687, 2024-09-21T09:43:24.044 INFO:teuthology.orchestra.run.smithi092.stdout: "old_weight": 0, 2024-09-21T09:43:24.044 INFO:teuthology.orchestra.run.smithi092.stdout: "last_purged_snaps_scrub": "2024-09-21T09:41:44.895848+0000", 2024-09-21T09:43:24.044 INFO:teuthology.orchestra.run.smithi092.stdout: "dead_epoch": 24 2024-09-21T09:43:24.044 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.044 INFO:teuthology.orchestra.run.smithi092.stdout: { 2024-09-21T09:43:24.045 INFO:teuthology.orchestra.run.smithi092.stdout: "osd": 3, 2024-09-21T09:43:24.045 INFO:teuthology.orchestra.run.smithi092.stdout: "down_stamp": "2024-09-21T09:43:13.277270+0000", 2024-09-21T09:43:24.045 INFO:teuthology.orchestra.run.smithi092.stdout: "laggy_probability": 0, 2024-09-21T09:43:24.045 INFO:teuthology.orchestra.run.smithi092.stdout: "laggy_interval": 0, 2024-09-21T09:43:24.045 INFO:teuthology.orchestra.run.smithi092.stdout: "features": 4540719139924082687, 2024-09-21T09:43:24.045 INFO:teuthology.orchestra.run.smithi092.stdout: "old_weight": 0, 2024-09-21T09:43:24.045 INFO:teuthology.orchestra.run.smithi092.stdout: "last_purged_snaps_scrub": "2024-09-21T09:41:44.636219+0000", 2024-09-21T09:43:24.045 INFO:teuthology.orchestra.run.smithi092.stdout: "dead_epoch": 0 2024-09-21T09:43:24.045 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.045 INFO:teuthology.orchestra.run.smithi092.stdout: ], 2024-09-21T09:43:24.045 INFO:teuthology.orchestra.run.smithi092.stdout: "pg_upmap": [], 2024-09-21T09:43:24.045 INFO:teuthology.orchestra.run.smithi092.stdout: "pg_upmap_items": [], 2024-09-21T09:43:24.045 INFO:teuthology.orchestra.run.smithi092.stdout: "pg_upmap_primaries": [], 2024-09-21T09:43:24.045 INFO:teuthology.orchestra.run.smithi092.stdout: "pg_temp": [], 2024-09-21T09:43:24.046 INFO:teuthology.orchestra.run.smithi092.stdout: "primary_temp": [], 2024-09-21T09:43:24.046 INFO:teuthology.orchestra.run.smithi092.stdout: "blocklist": { 2024-09-21T09:43:24.046 INFO:teuthology.orchestra.run.smithi092.stdout: "172.21.15.92:0/230938560": "2024-09-22T09:43:06.249869+0000", 2024-09-21T09:43:24.046 INFO:teuthology.orchestra.run.smithi092.stdout: "172.21.15.92:0/162460762": "2024-09-22T09:43:06.249869+0000", 2024-09-21T09:43:24.046 INFO:teuthology.orchestra.run.smithi092.stdout: "172.21.15.92:6800/1280404317": "2024-09-22T09:43:06.249869+0000", 2024-09-21T09:43:24.046 INFO:teuthology.orchestra.run.smithi092.stdout: "172.21.15.92:0/3587832392": "2024-09-22T09:43:06.249869+0000", 2024-09-21T09:43:24.046 INFO:teuthology.orchestra.run.smithi092.stdout: "172.21.15.92:0/2147030555": "2024-09-22T09:43:06.249869+0000" 2024-09-21T09:43:24.046 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.046 INFO:teuthology.orchestra.run.smithi092.stdout: "range_blocklist": {}, 2024-09-21T09:43:24.046 INFO:teuthology.orchestra.run.smithi092.stdout: "erasure_code_profiles": { 2024-09-21T09:43:24.046 INFO:teuthology.orchestra.run.smithi092.stdout: "default": { 2024-09-21T09:43:24.046 INFO:teuthology.orchestra.run.smithi092.stdout: "crush-failure-domain": "osd", 2024-09-21T09:43:24.046 INFO:teuthology.orchestra.run.smithi092.stdout: "k": "2", 2024-09-21T09:43:24.046 INFO:teuthology.orchestra.run.smithi092.stdout: "m": "1", 2024-09-21T09:43:24.047 INFO:teuthology.orchestra.run.smithi092.stdout: "plugin": "jerasure", 2024-09-21T09:43:24.047 INFO:teuthology.orchestra.run.smithi092.stdout: "technique": "reed_sol_van" 2024-09-21T09:43:24.047 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.047 INFO:teuthology.orchestra.run.smithi092.stdout: }, 2024-09-21T09:43:24.047 INFO:teuthology.orchestra.run.smithi092.stdout: "removed_snaps_queue": [], 2024-09-21T09:43:24.047 INFO:teuthology.orchestra.run.smithi092.stdout: "new_removed_snaps": [], 2024-09-21T09:43:24.047 INFO:teuthology.orchestra.run.smithi092.stdout: "new_purged_snaps": [], 2024-09-21T09:43:24.047 INFO:teuthology.orchestra.run.smithi092.stdout: "crush_node_flags": {}, 2024-09-21T09:43:24.047 INFO:teuthology.orchestra.run.smithi092.stdout: "device_class_flags": {}, 2024-09-21T09:43:24.047 INFO:teuthology.orchestra.run.smithi092.stdout: "stretch_mode": { 2024-09-21T09:43:24.047 INFO:teuthology.orchestra.run.smithi092.stdout: "stretch_mode_enabled": false, 2024-09-21T09:43:24.047 INFO:teuthology.orchestra.run.smithi092.stdout: "stretch_bucket_count": 0, 2024-09-21T09:43:24.047 INFO:teuthology.orchestra.run.smithi092.stdout: "degraded_stretch_mode": 0, 2024-09-21T09:43:24.047 INFO:teuthology.orchestra.run.smithi092.stdout: "recovering_stretch_mode": 0, 2024-09-21T09:43:24.047 INFO:teuthology.orchestra.run.smithi092.stdout: "stretch_mode_bucket": 0 2024-09-21T09:43:24.048 INFO:teuthology.orchestra.run.smithi092.stdout: } 2024-09-21T09:43:24.048 INFO:teuthology.orchestra.run.smithi092.stdout:} 2024-09-21T09:43:24.049 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.010+0000 7f705481d640 1 -- 172.21.15.92:0/959653741 >> v1:172.21.15.92:6800/2299163855 conn(0x7f70300514e0 legacy=0x7f70300539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:24.049 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.010+0000 7f705481d640 1 -- 172.21.15.92:0/959653741 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7050093b40 msgr2=0x7f7050070aa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:24.049 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.010+0000 7f705481d640 1 --2- 172.21.15.92:0/959653741 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7050093b40 0x7f7050070aa0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f703800ea60 tx=0x7f703800ef20 comp rx=0 tx=0).stop 2024-09-21T09:43:24.049 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.010+0000 7f705481d640 1 -- 172.21.15.92:0/959653741 shutdown_connections 2024-09-21T09:43:24.049 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.010+0000 7f705481d640 1 --2- 172.21.15.92:0/959653741 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7050094200 0x7f7050070fe0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.049 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.010+0000 7f705481d640 1 --2- 172.21.15.92:0/959653741 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7050093b40 0x7f7050070aa0 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.049 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.010+0000 7f705481d640 1 --2- 172.21.15.92:0/959653741 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7050093050 0x7f7050070560 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.050 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.010+0000 7f705481d640 1 -- 172.21.15.92:0/959653741 >> 172.21.15.92:0/959653741 conn(0x7f705009b580 msgr2=0x7f7050096ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:24.050 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.010+0000 7f705481d640 1 -- 172.21.15.92:0/959653741 shutdown_connections 2024-09-21T09:43:24.050 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.010+0000 7f705481d640 1 -- 172.21.15.92:0/959653741 wait complete. 2024-09-21T09:43:24.050 DEBUG:teuthology.orchestra.run.smithi092:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs dump' 2024-09-21T09:43:24.148 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.145+0000 7f5de55f9640 1 -- 172.21.15.92:0/2889856660 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5de009d060 msgr2=0x7f5de012e9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:24.148 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.145+0000 7f5de55f9640 1 --2- 172.21.15.92:0/2889856660 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5de009d060 0x7f5de012e9c0 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7f5dc8009860 tx=0x7f5dc802a3d0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.148 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.145+0000 7f5de55f9640 1 -- 172.21.15.92:0/2889856660 shutdown_connections 2024-09-21T09:43:24.148 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.145+0000 7f5de55f9640 1 --2- 172.21.15.92:0/2889856660 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5de009d060 0x7f5de012e9c0 unknown :-1 s=CLOSED pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.148 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.145+0000 7f5de55f9640 1 --2- 172.21.15.92:0/2889856660 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5de012b6b0 0x7f5de012ba90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.148 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.145+0000 7f5de55f9640 1 -- 172.21.15.92:0/2889856660 >> 172.21.15.92:0/2889856660 conn(0x7f5de006cf90 msgr2=0x7f5de006d3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:24.148 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.145+0000 7f5de55f9640 1 -- 172.21.15.92:0/2889856660 shutdown_connections 2024-09-21T09:43:24.148 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.145+0000 7f5de55f9640 1 -- 172.21.15.92:0/2889856660 wait complete. 2024-09-21T09:43:24.151 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.146+0000 7f5de55f9640 1 Processor -- start 2024-09-21T09:43:24.151 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.146+0000 7f5de55f9640 1 -- start start 2024-09-21T09:43:24.151 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.146+0000 7f5de55f9640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5de009d060 0x7f5de01323d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:24.151 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.146+0000 7f5de55f9640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5de012b6b0 0x7f5de0132910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:24.151 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.146+0000 7f5de55f9640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5de012bfd0 0x7f5de0132e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:24.151 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.146+0000 7f5de55f9640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f5de0137900 con 0x7f5de009d060 2024-09-21T09:43:24.151 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.146+0000 7f5de55f9640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f5de0137780 con 0x7f5de012b6b0 2024-09-21T09:43:24.151 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.146+0000 7f5de55f9640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f5de0137a80 con 0x7f5de012bfd0 2024-09-21T09:43:24.151 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.146+0000 7f5ddeffd640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5de009d060 0x7f5de01323d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:24.151 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.146+0000 7f5ddeffd640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5de009d060 0x7f5de01323d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:50662/0 (socket says 172.21.15.92:50662) 2024-09-21T09:43:24.151 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.146+0000 7f5ddeffd640 1 -- 172.21.15.92:0/2358555698 learned_addr learned my addr 172.21.15.92:0/2358555698 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:24.152 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.146+0000 7f5dde7fc640 1 --2- 172.21.15.92:0/2358555698 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5de012b6b0 0x7f5de0132910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:24.152 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.146+0000 7f5ddf7fe640 1 --2- 172.21.15.92:0/2358555698 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5de012bfd0 0x7f5de0132e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:24.152 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.146+0000 7f5ddeffd640 1 -- 172.21.15.92:0/2358555698 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5de012bfd0 msgr2=0x7f5de0132e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:24.152 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.146+0000 7f5ddeffd640 1 --2- 172.21.15.92:0/2358555698 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5de012bfd0 0x7f5de0132e50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.152 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.146+0000 7f5ddeffd640 1 -- 172.21.15.92:0/2358555698 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5de012b6b0 msgr2=0x7f5de0132910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:24.152 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.146+0000 7f5ddeffd640 1 --2- 172.21.15.92:0/2358555698 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5de012b6b0 0x7f5de0132910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.152 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.146+0000 7f5ddeffd640 1 -- 172.21.15.92:0/2358555698 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5de01346a0 con 0x7f5de009d060 2024-09-21T09:43:24.152 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.146+0000 7f5dde7fc640 1 --2- 172.21.15.92:0/2358555698 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5de012b6b0 0x7f5de0132910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:24.152 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.147+0000 7f5ddeffd640 1 --2- 172.21.15.92:0/2358555698 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5de009d060 0x7f5de01323d0 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f5dd800ea90 tx=0x7f5dd800ef50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:24.152 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.147+0000 7f5dd77fe640 1 -- 172.21.15.92:0/2358555698 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5dd800ce40 con 0x7f5de009d060 2024-09-21T09:43:24.152 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.147+0000 7f5dd77fe640 1 -- 172.21.15.92:0/2358555698 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f5dd8004020 con 0x7f5de009d060 2024-09-21T09:43:24.152 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.147+0000 7f5de55f9640 1 -- 172.21.15.92:0/2358555698 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5de0133ce0 con 0x7f5de009d060 2024-09-21T09:43:24.152 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.147+0000 7f5dd77fe640 1 -- 172.21.15.92:0/2358555698 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5dd8004470 con 0x7f5de009d060 2024-09-21T09:43:24.152 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.147+0000 7f5de55f9640 1 -- 172.21.15.92:0/2358555698 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5de0133570 con 0x7f5de009d060 2024-09-21T09:43:24.152 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.148+0000 7f5de55f9640 1 -- 172.21.15.92:0/2358555698 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5de012ba90 con 0x7f5de009d060 2024-09-21T09:43:24.153 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.148+0000 7f5dd77fe640 1 -- 172.21.15.92:0/2358555698 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f5dd80035f0 con 0x7f5de009d060 2024-09-21T09:43:24.153 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.149+0000 7f5dd77fe640 1 -- 172.21.15.92:0/2358555698 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(31..31 src has 1..31) ==== 3027+0+0 (secure 0 0 0) 0x7f5dd8069940 con 0x7f5de009d060 2024-09-21T09:43:24.155 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.152+0000 7f5dd77fe640 1 -- 172.21.15.92:0/2358555698 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f5dd80689f0 con 0x7f5de009d060 2024-09-21T09:43:24.332 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.329+0000 7f5de55f9640 1 -- 172.21.15.92:0/2358555698 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "fs dump"} v 0) -- 0x7f5de01274c0 con 0x7f5de009d060 2024-09-21T09:43:24.332 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.329+0000 7f5dd77fe640 1 -- 172.21.15.92:0/2358555698 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump"}]=0 dumped fsmap epoch 1 v1) ==== 75+0+440 (secure 0 0 0) 0x7f5dd8068530 con 0x7f5de009d060 2024-09-21T09:43:24.332 INFO:teuthology.orchestra.run.smithi092.stdout:e1 2024-09-21T09:43:24.332 INFO:teuthology.orchestra.run.smithi092.stdout:btime 2024-09-21T09:41:34:677977+0000 2024-09-21T09:43:24.332 INFO:teuthology.orchestra.run.smithi092.stdout:enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:43:24.332 INFO:teuthology.orchestra.run.smithi092.stdout:default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2,11=minor log segments,12=quiesce subvolumes} 2024-09-21T09:43:24.332 INFO:teuthology.orchestra.run.smithi092.stdout:legacy client fscid: -1 2024-09-21T09:43:24.332 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:24.332 INFO:teuthology.orchestra.run.smithi092.stdout:No filesystems configured 2024-09-21T09:43:24.333 INFO:teuthology.orchestra.run.smithi092.stderr:dumped fsmap epoch 1 2024-09-21T09:43:24.333 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.331+0000 7f5de55f9640 1 -- 172.21.15.92:0/2358555698 >> v1:172.21.15.92:6800/2299163855 conn(0x7f5dc00514b0 legacy=0x7f5dc0053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:24.333 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.331+0000 7f5de55f9640 1 -- 172.21.15.92:0/2358555698 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5de009d060 msgr2=0x7f5de01323d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:24.334 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.331+0000 7f5de55f9640 1 --2- 172.21.15.92:0/2358555698 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5de009d060 0x7f5de01323d0 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f5dd800ea90 tx=0x7f5dd800ef50 comp rx=0 tx=0).stop 2024-09-21T09:43:24.334 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.331+0000 7f5de55f9640 1 -- 172.21.15.92:0/2358555698 shutdown_connections 2024-09-21T09:43:24.334 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.331+0000 7f5de55f9640 1 --2- 172.21.15.92:0/2358555698 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5de012bfd0 0x7f5de0132e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.334 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.331+0000 7f5de55f9640 1 --2- 172.21.15.92:0/2358555698 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5de012b6b0 0x7f5de0132910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.334 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.331+0000 7f5de55f9640 1 --2- 172.21.15.92:0/2358555698 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5de009d060 0x7f5de01323d0 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.334 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.331+0000 7f5de55f9640 1 -- 172.21.15.92:0/2358555698 >> 172.21.15.92:0/2358555698 conn(0x7f5de006cf90 msgr2=0x7f5de009fd20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:24.334 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.331+0000 7f5de55f9640 1 -- 172.21.15.92:0/2358555698 shutdown_connections 2024-09-21T09:43:24.334 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.331+0000 7f5de55f9640 1 -- 172.21.15.92:0/2358555698 wait complete. 2024-09-21T09:43:24.344 DEBUG:teuthology.orchestra.run.smithi092:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph osd require-osd-release squid' 2024-09-21T09:43:24.444 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.440+0000 7f718768c640 1 -- 172.21.15.92:0/3857182115 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f71800947f0 msgr2=0x7f7180094bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:24.444 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.440+0000 7f718768c640 1 --2- 172.21.15.92:0/3857182115 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f71800947f0 0x7f7180094bd0 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f717c009c10 tx=0x7f717c02a780 comp rx=0 tx=0).stop 2024-09-21T09:43:24.444 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.440+0000 7f718768c640 1 -- 172.21.15.92:0/3857182115 shutdown_connections 2024-09-21T09:43:24.444 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.440+0000 7f718768c640 1 --2- 172.21.15.92:0/3857182115 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7180095550 0x7f71801329b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.444 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.440+0000 7f718768c640 1 --2- 172.21.15.92:0/3857182115 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f71800947f0 0x7f7180094bd0 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.444 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.440+0000 7f718768c640 1 -- 172.21.15.92:0/3857182115 >> 172.21.15.92:0/3857182115 conn(0x7f718009a6e0 msgr2=0x7f7180097b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:24.444 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.440+0000 7f718768c640 1 -- 172.21.15.92:0/3857182115 shutdown_connections 2024-09-21T09:43:24.444 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.440+0000 7f718768c640 1 -- 172.21.15.92:0/3857182115 wait complete. 2024-09-21T09:43:24.444 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.441+0000 7f718768c640 1 Processor -- start 2024-09-21T09:43:24.444 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.441+0000 7f718768c640 1 -- start start 2024-09-21T09:43:24.447 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.441+0000 7f718768c640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f71800947f0 0x7f7180131700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:24.447 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.441+0000 7f718768c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7180095110 0x7f7180131c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:24.447 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.441+0000 7f718768c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7180095550 0x7f7180132180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:24.447 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.441+0000 7f718768c640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f71801335e0 con 0x7f7180095110 2024-09-21T09:43:24.447 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.441+0000 7f718768c640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f7180133460 con 0x7f7180095550 2024-09-21T09:43:24.447 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.441+0000 7f718768c640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f7180133760 con 0x7f71800947f0 2024-09-21T09:43:24.447 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.441+0000 7f7185c29640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7180095550 0x7f7180132180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:24.447 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.441+0000 7f7185428640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f71800947f0 0x7f7180131700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:24.447 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.442+0000 7f7184c27640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7180095110 0x7f7180131c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:24.447 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.442+0000 7f7185428640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f71800947f0 0x7f7180131700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:42296/0 (socket says 172.21.15.92:42296) 2024-09-21T09:43:24.447 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.442+0000 7f7185428640 1 -- 172.21.15.92:0/453057114 learned_addr learned my addr 172.21.15.92:0/453057114 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:24.447 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.442+0000 7f7185c29640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7180095550 0x7f7180132180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:43772/0 (socket says 172.21.15.92:43772) 2024-09-21T09:43:24.448 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.442+0000 7f7185428640 1 -- 172.21.15.92:0/453057114 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7180095550 msgr2=0x7f7180132180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:24.448 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.442+0000 7f7185428640 1 --2- 172.21.15.92:0/453057114 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7180095550 0x7f7180132180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.448 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.442+0000 7f7185428640 1 -- 172.21.15.92:0/453057114 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7180095110 msgr2=0x7f7180131c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:24.448 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.442+0000 7f7185428640 1 --2- 172.21.15.92:0/453057114 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7180095110 0x7f7180131c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.448 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.442+0000 7f7185428640 1 -- 172.21.15.92:0/453057114 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7180134a00 con 0x7f71800947f0 2024-09-21T09:43:24.448 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.442+0000 7f7184c27640 1 --2- 172.21.15.92:0/453057114 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7180095110 0x7f7180131c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:24.448 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.442+0000 7f7185c29640 1 --2- 172.21.15.92:0/453057114 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7180095550 0x7f7180132180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:24.448 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.442+0000 7f7185428640 1 --2- 172.21.15.92:0/453057114 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f71800947f0 0x7f7180131700 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f717c009d30 tx=0x7f717c032f10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:24.448 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.442+0000 7f7175ffb640 1 -- 172.21.15.92:0/453057114 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f717c041020 con 0x7f71800947f0 2024-09-21T09:43:24.448 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.442+0000 7f7175ffb640 1 -- 172.21.15.92:0/453057114 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f717c004130 con 0x7f71800947f0 2024-09-21T09:43:24.448 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.442+0000 7f7175ffb640 1 -- 172.21.15.92:0/453057114 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f717c036040 con 0x7f71800947f0 2024-09-21T09:43:24.448 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.442+0000 7f718768c640 1 -- 172.21.15.92:0/453057114 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f71801233b0 con 0x7f71800947f0 2024-09-21T09:43:24.449 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.442+0000 7f718768c640 1 -- 172.21.15.92:0/453057114 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f7180123830 con 0x7f71800947f0 2024-09-21T09:43:24.449 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.443+0000 7f718768c640 1 -- 172.21.15.92:0/453057114 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7180094bd0 con 0x7f71800947f0 2024-09-21T09:43:24.449 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.443+0000 7f7175ffb640 1 -- 172.21.15.92:0/453057114 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f717c044020 con 0x7f71800947f0 2024-09-21T09:43:24.449 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.444+0000 7f7175ffb640 1 -- 172.21.15.92:0/453057114 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(31..31 src has 1..31) ==== 3027+0+0 (secure 0 0 0) 0x7f717c089fa0 con 0x7f71800947f0 2024-09-21T09:43:24.450 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.447+0000 7f7175ffb640 1 -- 172.21.15.92:0/453057114 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f717c04b030 con 0x7f71800947f0 2024-09-21T09:43:24.605 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.602+0000 7f718768c640 1 -- 172.21.15.92:0/453057114 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "osd require-osd-release", "release": "squid"} v 0) -- 0x7f718006a3e0 con 0x7f71800947f0 2024-09-21T09:43:24.605 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.603+0000 7f7175ffb640 1 -- 172.21.15.92:0/453057114 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mon_command_ack([{"prefix": "osd require-osd-release", "release": "squid"}]=0 v31) ==== 91+0+0 (secure 0 0 0) 0x7f717c089050 con 0x7f71800947f0 2024-09-21T09:43:24.608 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.605+0000 7f718768c640 1 -- 172.21.15.92:0/453057114 >> v1:172.21.15.92:6800/2299163855 conn(0x7f71640514b0 legacy=0x7f7164053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:24.608 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.605+0000 7f718768c640 1 -- 172.21.15.92:0/453057114 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f71800947f0 msgr2=0x7f7180131700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:24.608 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.605+0000 7f718768c640 1 --2- 172.21.15.92:0/453057114 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f71800947f0 0x7f7180131700 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f717c009d30 tx=0x7f717c032f10 comp rx=0 tx=0).stop 2024-09-21T09:43:24.608 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.605+0000 7f718768c640 1 -- 172.21.15.92:0/453057114 shutdown_connections 2024-09-21T09:43:24.608 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.605+0000 7f718768c640 1 --2- 172.21.15.92:0/453057114 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7180095550 0x7f7180132180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.608 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.605+0000 7f718768c640 1 --2- 172.21.15.92:0/453057114 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7180095110 0x7f7180131c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.608 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.605+0000 7f718768c640 1 --2- 172.21.15.92:0/453057114 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f71800947f0 0x7f7180131700 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.608 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.605+0000 7f718768c640 1 -- 172.21.15.92:0/453057114 >> 172.21.15.92:0/453057114 conn(0x7f718009a6e0 msgr2=0x7f718009af80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:24.608 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.605+0000 7f718768c640 1 -- 172.21.15.92:0/453057114 shutdown_connections 2024-09-21T09:43:24.608 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.605+0000 7f718768c640 1 -- 172.21.15.92:0/453057114 wait complete. 2024-09-21T09:43:24.618 DEBUG:teuthology.orchestra.run.smithi092:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'for f in `ceph osd pool ls` ; do ceph osd pool set $f pg_autoscale_mode off ; done' 2024-09-21T09:43:24.717 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.714+0000 7f6ccedd5640 1 -- 172.21.15.92:0/807358049 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6cc8093880 msgr2=0x7f6cc8131be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:24.717 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.714+0000 7f6ccedd5640 1 --2- 172.21.15.92:0/807358049 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6cc8093880 0x7f6cc8131be0 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f6cbc009860 tx=0x7f6cbc02a4b0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.717 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.714+0000 7f6ccedd5640 1 -- 172.21.15.92:0/807358049 shutdown_connections 2024-09-21T09:43:24.717 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.714+0000 7f6ccedd5640 1 --2- 172.21.15.92:0/807358049 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6cc8132120 0x7f6cc8134510 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.717 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.714+0000 7f6ccedd5640 1 --2- 172.21.15.92:0/807358049 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6cc8093880 0x7f6cc8131be0 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.717 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.714+0000 7f6ccedd5640 1 -- 172.21.15.92:0/807358049 >> 172.21.15.92:0/807358049 conn(0x7f6cc809a6e0 msgr2=0x7f6cc8097b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:24.717 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.714+0000 7f6ccedd5640 1 -- 172.21.15.92:0/807358049 shutdown_connections 2024-09-21T09:43:24.717 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.714+0000 7f6ccedd5640 1 -- 172.21.15.92:0/807358049 wait complete. 2024-09-21T09:43:24.720 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.715+0000 7f6ccedd5640 1 Processor -- start 2024-09-21T09:43:24.720 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.715+0000 7f6ccedd5640 1 -- start start 2024-09-21T09:43:24.720 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.715+0000 7f6ccedd5640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6cc8093880 0x7f6cc8091110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:24.720 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.715+0000 7f6ccedd5640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6cc812cbb0 0x7f6cc8091650 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:24.720 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.715+0000 7f6ccedd5640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6cc8132120 0x7f6cc8091b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:24.720 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.715+0000 7f6ccedd5640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f6cc81356f0 con 0x7f6cc812cbb0 2024-09-21T09:43:24.720 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.715+0000 7f6ccedd5640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f6cc8135570 con 0x7f6cc8132120 2024-09-21T09:43:24.720 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.715+0000 7f6ccedd5640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f6cc8135870 con 0x7f6cc8093880 2024-09-21T09:43:24.720 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.715+0000 7f6cccb71640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6cc8093880 0x7f6cc8091110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:24.720 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.715+0000 7f6cc7fff640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6cc812cbb0 0x7f6cc8091650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:24.720 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.715+0000 7f6cccb71640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6cc8093880 0x7f6cc8091110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:42308/0 (socket says 172.21.15.92:42308) 2024-09-21T09:43:24.720 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.715+0000 7f6ccd372640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6cc8132120 0x7f6cc8091b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:24.721 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.715+0000 7f6cccb71640 1 -- 172.21.15.92:0/2641808150 learned_addr learned my addr 172.21.15.92:0/2641808150 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:24.721 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.715+0000 7f6cc7fff640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6cc812cbb0 0x7f6cc8091650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:50690/0 (socket says 172.21.15.92:50690) 2024-09-21T09:43:24.721 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.716+0000 7f6cccb71640 1 -- 172.21.15.92:0/2641808150 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6cc8132120 msgr2=0x7f6cc8091b90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:24.721 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.716+0000 7f6cccb71640 1 --2- 172.21.15.92:0/2641808150 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6cc8132120 0x7f6cc8091b90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.721 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.716+0000 7f6cccb71640 1 -- 172.21.15.92:0/2641808150 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6cc812cbb0 msgr2=0x7f6cc8091650 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:24.721 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.716+0000 7f6cccb71640 1 --2- 172.21.15.92:0/2641808150 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6cc812cbb0 0x7f6cc8091650 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.721 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.716+0000 7f6cccb71640 1 -- 172.21.15.92:0/2641808150 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6cc8092320 con 0x7f6cc8093880 2024-09-21T09:43:24.721 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.716+0000 7f6cc7fff640 1 --2- 172.21.15.92:0/2641808150 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6cc812cbb0 0x7f6cc8091650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:24.721 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.716+0000 7f6ccd372640 1 --2- 172.21.15.92:0/2641808150 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6cc8132120 0x7f6cc8091b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:24.721 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.716+0000 7f6cccb71640 1 --2- 172.21.15.92:0/2641808150 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6cc8093880 0x7f6cc8091110 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f6cb800c530 tx=0x7f6cb800c9f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:24.721 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.716+0000 7f6cc57fa640 1 -- 172.21.15.92:0/2641808150 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6cb8016020 con 0x7f6cc8093880 2024-09-21T09:43:24.721 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.716+0000 7f6cc57fa640 1 -- 172.21.15.92:0/2641808150 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f6cb8012920 con 0x7f6cc8093880 2024-09-21T09:43:24.721 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.716+0000 7f6cc57fa640 1 -- 172.21.15.92:0/2641808150 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6cb8012d50 con 0x7f6cc8093880 2024-09-21T09:43:24.721 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.716+0000 7f6ccedd5640 1 -- 172.21.15.92:0/2641808150 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6cc8090540 con 0x7f6cc8093880 2024-09-21T09:43:24.722 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.716+0000 7f6ccedd5640 1 -- 172.21.15.92:0/2641808150 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f6cc8092520 con 0x7f6cc8093880 2024-09-21T09:43:24.722 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.717+0000 7f6cc57fa640 1 -- 172.21.15.92:0/2641808150 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f6cb8019020 con 0x7f6cc8093880 2024-09-21T09:43:24.722 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.717+0000 7f6ccedd5640 1 -- 172.21.15.92:0/2641808150 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6cc812cf90 con 0x7f6cc8093880 2024-09-21T09:43:24.722 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.718+0000 7f6cc57fa640 1 -- 172.21.15.92:0/2641808150 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(31..31 src has 1..31) ==== 3027+0+0 (secure 0 0 0) 0x7f6cb8068bd0 con 0x7f6cc8093880 2024-09-21T09:43:24.724 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.721+0000 7f6cc57fa640 1 -- 172.21.15.92:0/2641808150 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f6cb8020030 con 0x7f6cc8093880 2024-09-21T09:43:24.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.871+0000 7f6ccedd5640 1 -- 172.21.15.92:0/2641808150 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "osd pool ls"} v 0) -- 0x7f6cc806a3e0 con 0x7f6cc8093880 2024-09-21T09:43:24.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.871+0000 7f6cc57fa640 1 -- 172.21.15.92:0/2641808150 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mon_command_ack([{"prefix": "osd pool ls"}]=0 v31) ==== 59+0+9 (secure 0 0 0) 0x7f6cb8067c80 con 0x7f6cc8093880 2024-09-21T09:43:24.876 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.873+0000 7f6ccedd5640 1 -- 172.21.15.92:0/2641808150 >> v1:172.21.15.92:6800/2299163855 conn(0x7f6cb40514b0 legacy=0x7f6cb4053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:24.876 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.873+0000 7f6ccedd5640 1 -- 172.21.15.92:0/2641808150 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6cc8093880 msgr2=0x7f6cc8091110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:24.876 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.873+0000 7f6ccedd5640 1 --2- 172.21.15.92:0/2641808150 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6cc8093880 0x7f6cc8091110 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f6cb800c530 tx=0x7f6cb800c9f0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.876 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.873+0000 7f6ccedd5640 1 -- 172.21.15.92:0/2641808150 shutdown_connections 2024-09-21T09:43:24.876 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.873+0000 7f6ccedd5640 1 --2- 172.21.15.92:0/2641808150 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6cc8132120 0x7f6cc8091b90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.877 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.873+0000 7f6ccedd5640 1 --2- 172.21.15.92:0/2641808150 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6cc812cbb0 0x7f6cc8091650 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.877 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.873+0000 7f6ccedd5640 1 --2- 172.21.15.92:0/2641808150 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6cc8093880 0x7f6cc8091110 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.877 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.873+0000 7f6ccedd5640 1 -- 172.21.15.92:0/2641808150 >> 172.21.15.92:0/2641808150 conn(0x7f6cc809a6e0 msgr2=0x7f6cc8097b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:24.877 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.873+0000 7f6ccedd5640 1 -- 172.21.15.92:0/2641808150 shutdown_connections 2024-09-21T09:43:24.877 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.873+0000 7f6ccedd5640 1 -- 172.21.15.92:0/2641808150 wait complete. 2024-09-21T09:43:24.964 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.960+0000 7fc1155e7640 1 -- 172.21.15.92:0/3260596568 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc110094b10 msgr2=0x7fc110094f70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:24.964 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.960+0000 7fc1155e7640 1 --2- 172.21.15.92:0/3260596568 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc110094b10 0x7fc110094f70 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7fc0f8005fd0 tx=0x7fc0f802a410 comp rx=0 tx=0).stop 2024-09-21T09:43:24.964 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.960+0000 7fc1155e7640 1 -- 172.21.15.92:0/3260596568 shutdown_connections 2024-09-21T09:43:24.964 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.960+0000 7fc1155e7640 1 --2- 172.21.15.92:0/3260596568 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc1100954b0 0x7fc110133b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.964 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.960+0000 7fc1155e7640 1 --2- 172.21.15.92:0/3260596568 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc110094b10 0x7fc110094f70 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.964 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.960+0000 7fc1155e7640 1 --2- 172.21.15.92:0/3260596568 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc11012cd60 0x7fc11012d140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.964 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.960+0000 7fc1155e7640 1 -- 172.21.15.92:0/3260596568 >> 172.21.15.92:0/3260596568 conn(0x7fc11009d640 msgr2=0x7fc11009da50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:24.964 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.960+0000 7fc1155e7640 1 -- 172.21.15.92:0/3260596568 shutdown_connections 2024-09-21T09:43:24.964 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.960+0000 7fc1155e7640 1 -- 172.21.15.92:0/3260596568 wait complete. 2024-09-21T09:43:24.964 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.961+0000 7fc1155e7640 1 Processor -- start 2024-09-21T09:43:24.964 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.961+0000 7fc1155e7640 1 -- start start 2024-09-21T09:43:24.966 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.961+0000 7fc1155e7640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc110094b10 0x7fc1101c4cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:24.967 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.961+0000 7fc1155e7640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc1100954b0 0x7fc1101c5210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:24.967 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.961+0000 7fc1155e7640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc11012cd60 0x7fc110131cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:24.967 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.961+0000 7fc1155e7640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fc110134670 con 0x7fc1100954b0 2024-09-21T09:43:24.967 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.961+0000 7fc1155e7640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fc1101344f0 con 0x7fc110094b10 2024-09-21T09:43:24.967 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.961+0000 7fc1155e7640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fc1101347f0 con 0x7fc11012cd60 2024-09-21T09:43:24.967 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.962+0000 7fc107fff640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc1100954b0 0x7fc1101c5210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:24.967 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.962+0000 7fc10f7fe640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc11012cd60 0x7fc110131cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:24.967 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.962+0000 7fc10effd640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc110094b10 0x7fc1101c4cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:24.967 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.962+0000 7fc107fff640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc1100954b0 0x7fc1101c5210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:50718/0 (socket says 172.21.15.92:50718) 2024-09-21T09:43:24.967 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.962+0000 7fc107fff640 1 -- 172.21.15.92:0/96300900 learned_addr learned my addr 172.21.15.92:0/96300900 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:24.967 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.962+0000 7fc10f7fe640 1 -- 172.21.15.92:0/96300900 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc110094b10 msgr2=0x7fc1101c4cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:24.967 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.962+0000 7fc10f7fe640 1 --2- 172.21.15.92:0/96300900 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc110094b10 0x7fc1101c4cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.968 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.962+0000 7fc10f7fe640 1 -- 172.21.15.92:0/96300900 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc1100954b0 msgr2=0x7fc1101c5210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:24.968 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.962+0000 7fc10f7fe640 1 --2- 172.21.15.92:0/96300900 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc1100954b0 0x7fc1101c5210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:24.968 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.962+0000 7fc10f7fe640 1 -- 172.21.15.92:0/96300900 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc110133600 con 0x7fc11012cd60 2024-09-21T09:43:24.968 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.962+0000 7fc10f7fe640 1 --2- 172.21.15.92:0/96300900 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc11012cd60 0x7fc110131cc0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7fc10800eb70 tx=0x7fc10800c6a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:24.968 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.962+0000 7fc1077fe640 1 -- 172.21.15.92:0/96300900 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc108003b70 con 0x7fc11012cd60 2024-09-21T09:43:24.968 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.962+0000 7fc1077fe640 1 -- 172.21.15.92:0/96300900 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fc108003d10 con 0x7fc11012cd60 2024-09-21T09:43:24.968 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.962+0000 7fc1155e7640 1 -- 172.21.15.92:0/96300900 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc1100975f0 con 0x7fc11012cd60 2024-09-21T09:43:24.968 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.962+0000 7fc1077fe640 1 -- 172.21.15.92:0/96300900 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc108004280 con 0x7fc11012cd60 2024-09-21T09:43:24.968 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.962+0000 7fc1155e7640 1 -- 172.21.15.92:0/96300900 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fc110091610 con 0x7fc11012cd60 2024-09-21T09:43:24.968 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.963+0000 7fc1155e7640 1 -- 172.21.15.92:0/96300900 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc110098440 con 0x7fc11012cd60 2024-09-21T09:43:24.968 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.963+0000 7fc1077fe640 1 -- 172.21.15.92:0/96300900 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7fc108004420 con 0x7fc11012cd60 2024-09-21T09:43:24.968 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.964+0000 7fc1077fe640 1 -- 172.21.15.92:0/96300900 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(31..31 src has 1..31) ==== 3027+0+0 (secure 0 0 0) 0x7fc108069bd0 con 0x7fc11012cd60 2024-09-21T09:43:24.970 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:24.967+0000 7fc1077fe640 1 -- 172.21.15.92:0/96300900 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fc110098440 con 0x7fc11012cd60 2024-09-21T09:43:25.121 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.118+0000 7fc1155e7640 1 -- 172.21.15.92:0/96300900 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "osd pool set", "pool": ".mgr", "var": "pg_autoscale_mode", "val": "off"} v 0) -- 0x7fc110094f70 con 0x7fc11012cd60 2024-09-21T09:43:25.336 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.333+0000 7fc1077fe640 1 -- 172.21.15.92:0/96300900 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mon_command_ack([{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_autoscale_mode", "val": "off"}]=0 set pool 1 pg_autoscale_mode to off v32) ==== 153+0+0 (secure 0 0 0) 0x7fc108068c80 con 0x7fc11012cd60 2024-09-21T09:43:25.336 INFO:teuthology.orchestra.run.smithi092.stderr:set pool 1 pg_autoscale_mode to off 2024-09-21T09:43:25.339 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.335+0000 7fc1155e7640 1 -- 172.21.15.92:0/96300900 >> v1:172.21.15.92:6800/2299163855 conn(0x7fc0f00514b0 legacy=0x7fc0f0053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:25.339 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.335+0000 7fc1155e7640 1 -- 172.21.15.92:0/96300900 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc11012cd60 msgr2=0x7fc110131cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:25.339 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.335+0000 7fc1155e7640 1 --2- 172.21.15.92:0/96300900 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc11012cd60 0x7fc110131cc0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7fc10800eb70 tx=0x7fc10800c6a0 comp rx=0 tx=0).stop 2024-09-21T09:43:25.339 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.336+0000 7fc1155e7640 1 -- 172.21.15.92:0/96300900 shutdown_connections 2024-09-21T09:43:25.339 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.336+0000 7fc1155e7640 1 --2- 172.21.15.92:0/96300900 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc11012cd60 0x7fc110131cc0 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:25.339 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.336+0000 7fc1155e7640 1 --2- 172.21.15.92:0/96300900 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc1100954b0 0x7fc1101c5210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:25.339 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.336+0000 7fc1155e7640 1 --2- 172.21.15.92:0/96300900 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc110094b10 0x7fc1101c4cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:25.339 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.336+0000 7fc1155e7640 1 -- 172.21.15.92:0/96300900 >> 172.21.15.92:0/96300900 conn(0x7fc11009d640 msgr2=0x7fc11012c020 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:25.339 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.336+0000 7fc1155e7640 1 -- 172.21.15.92:0/96300900 shutdown_connections 2024-09-21T09:43:25.339 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.336+0000 7fc1155e7640 1 -- 172.21.15.92:0/96300900 wait complete. 2024-09-21T09:43:25.426 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.423+0000 7f56297a2640 1 -- 172.21.15.92:0/3864901338 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5624096f70 msgr2=0x7f5624097420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:25.426 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.423+0000 7f56297a2640 1 --2- 172.21.15.92:0/3864901338 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5624096f70 0x7f5624097420 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f561c005fd0 tx=0x7f561c02a6e0 comp rx=0 tx=0).stop 2024-09-21T09:43:25.426 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.423+0000 7f56297a2640 1 -- 172.21.15.92:0/3864901338 shutdown_connections 2024-09-21T09:43:25.426 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.423+0000 7f56297a2640 1 --2- 172.21.15.92:0/3864901338 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5624096f70 0x7f5624097420 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:25.426 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.423+0000 7f56297a2640 1 --2- 172.21.15.92:0/3864901338 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5624071920 0x7f5624070fb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:25.426 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.423+0000 7f56297a2640 1 --2- 172.21.15.92:0/3864901338 >> v2:172.21.15.92:3300/0 conn(0x7f5624070690 0x7f5624070a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:25.426 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.423+0000 7f56297a2640 1 -- 172.21.15.92:0/3864901338 >> 172.21.15.92:0/3864901338 conn(0x7f562409a320 msgr2=0x7f562409a730 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:25.427 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.423+0000 7f56297a2640 1 -- 172.21.15.92:0/3864901338 shutdown_connections 2024-09-21T09:43:25.427 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.423+0000 7f56297a2640 1 -- 172.21.15.92:0/3864901338 wait complete. 2024-09-21T09:43:25.431 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.424+0000 7f56297a2640 1 Processor -- start 2024-09-21T09:43:25.431 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.424+0000 7f56297a2640 1 -- start start 2024-09-21T09:43:25.431 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.424+0000 7f56297a2640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5624070690 0x7f5624128750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:25.431 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.424+0000 7f56297a2640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5624071920 0x7f5624128c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:25.431 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.424+0000 7f56297a2640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5624096f70 0x7f56241291d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:25.431 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.424+0000 7f56297a2640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f5624133610 con 0x7f5624070690 2024-09-21T09:43:25.431 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.424+0000 7f56297a2640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f5624133490 con 0x7f5624071920 2024-09-21T09:43:25.431 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.424+0000 7f56297a2640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f5624133790 con 0x7f5624096f70 2024-09-21T09:43:25.432 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.424+0000 7f5623fff640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5624070690 0x7f5624128750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:25.432 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.424+0000 7f5623fff640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5624070690 0x7f5624128750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:50736/0 (socket says 172.21.15.92:50736) 2024-09-21T09:43:25.432 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.424+0000 7f5623fff640 1 -- 172.21.15.92:0/896413761 learned_addr learned my addr 172.21.15.92:0/896413761 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:25.432 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.424+0000 7f56237fe640 1 --2- 172.21.15.92:0/896413761 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5624071920 0x7f5624128c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:25.432 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.424+0000 7f5628fa1640 1 --2- 172.21.15.92:0/896413761 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5624096f70 0x7f56241291d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:25.432 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.425+0000 7f5623fff640 1 -- 172.21.15.92:0/896413761 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5624096f70 msgr2=0x7f56241291d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:25.432 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.425+0000 7f5623fff640 1 --2- 172.21.15.92:0/896413761 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5624096f70 0x7f56241291d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:25.432 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.425+0000 7f5623fff640 1 -- 172.21.15.92:0/896413761 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5624071920 msgr2=0x7f5624128c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:25.432 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.425+0000 7f5623fff640 1 --2- 172.21.15.92:0/896413761 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5624071920 0x7f5624128c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:25.432 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.425+0000 7f5623fff640 1 -- 172.21.15.92:0/896413761 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f562406f8d0 con 0x7f5624070690 2024-09-21T09:43:25.432 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.425+0000 7f56237fe640 1 --2- 172.21.15.92:0/896413761 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5624071920 0x7f5624128c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:25.432 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.425+0000 7f5623fff640 1 --2- 172.21.15.92:0/896413761 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5624070690 0x7f5624128750 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f561400e9c0 tx=0x7f561400ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:25.432 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.425+0000 7f5620ff9640 1 -- 172.21.15.92:0/896413761 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f561400cde0 con 0x7f5624070690 2024-09-21T09:43:25.433 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.425+0000 7f5620ff9640 1 -- 172.21.15.92:0/896413761 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f5614004020 con 0x7f5624070690 2024-09-21T09:43:25.433 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.425+0000 7f56297a2640 1 -- 172.21.15.92:0/896413761 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f562412a390 con 0x7f5624070690 2024-09-21T09:43:25.433 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.425+0000 7f5620ff9640 1 -- 172.21.15.92:0/896413761 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5614004450 con 0x7f5624070690 2024-09-21T09:43:25.433 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.425+0000 7f56297a2640 1 -- 172.21.15.92:0/896413761 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f562412ac60 con 0x7f5624070690 2024-09-21T09:43:25.433 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.426+0000 7f56297a2640 1 -- 172.21.15.92:0/896413761 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5624097420 con 0x7f5624070690 2024-09-21T09:43:25.433 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.426+0000 7f5620ff9640 1 -- 172.21.15.92:0/896413761 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f56140035f0 con 0x7f5624070690 2024-09-21T09:43:25.433 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.426+0000 7f5620ff9640 1 -- 172.21.15.92:0/896413761 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7f5614069990 con 0x7f5624070690 2024-09-21T09:43:25.433 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.430+0000 7f5620ff9640 1 -- 172.21.15.92:0/896413761 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f5614068a40 con 0x7f5624070690 2024-09-21T09:43:25.581 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:25.578+0000 7f56297a2640 1 -- 172.21.15.92:0/896413761 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd pool set", "pool": "rbd", "var": "pg_autoscale_mode", "val": "off"} v 0) -- 0x7f56240008d0 con 0x7f5624070690 2024-09-21T09:43:26.347 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.345+0000 7f5620ff9640 1 -- 172.21.15.92:0/896413761 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool set", "pool": "rbd", "var": "pg_autoscale_mode", "val": "off"}]=0 set pool 2 pg_autoscale_mode to off v33) ==== 152+0+0 (secure 0 0 0) 0x7f5614009460 con 0x7f5624070690 2024-09-21T09:43:26.347 INFO:teuthology.orchestra.run.smithi092.stderr:set pool 2 pg_autoscale_mode to off 2024-09-21T09:43:26.350 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.348+0000 7f56297a2640 1 -- 172.21.15.92:0/896413761 >> v1:172.21.15.92:6800/2299163855 conn(0x7f56040514b0 legacy=0x7f5604053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:26.350 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.348+0000 7f56297a2640 1 -- 172.21.15.92:0/896413761 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5624070690 msgr2=0x7f5624128750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:26.350 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.348+0000 7f56297a2640 1 --2- 172.21.15.92:0/896413761 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5624070690 0x7f5624128750 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f561400e9c0 tx=0x7f561400ee80 comp rx=0 tx=0).stop 2024-09-21T09:43:26.350 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.348+0000 7f56297a2640 1 -- 172.21.15.92:0/896413761 shutdown_connections 2024-09-21T09:43:26.351 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.348+0000 7f56297a2640 1 --2- 172.21.15.92:0/896413761 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f5624096f70 0x7f56241291d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:26.351 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.348+0000 7f56297a2640 1 --2- 172.21.15.92:0/896413761 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f5624071920 0x7f5624128c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:26.351 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.348+0000 7f56297a2640 1 --2- 172.21.15.92:0/896413761 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f5624070690 0x7f5624128750 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:26.351 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.348+0000 7f56297a2640 1 -- 172.21.15.92:0/896413761 >> 172.21.15.92:0/896413761 conn(0x7f562409a320 msgr2=0x7f5624098290 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:26.351 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.348+0000 7f56297a2640 1 -- 172.21.15.92:0/896413761 shutdown_connections 2024-09-21T09:43:26.351 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.349+0000 7f56297a2640 1 -- 172.21.15.92:0/896413761 wait complete. 2024-09-21T09:43:26.364 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-21T09:43:26.374 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T09:43:26.374 INFO:tasks.ceph.ceph_manager.ceph:waiting for mgr available 2024-09-21T09:43:26.374 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mgr dump --format=json 2024-09-21T09:43:26.480 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.478+0000 7f0ad108c640 1 -- 172.21.15.92:0/582212692 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0acc12b490 msgr2=0x7f0acc12b870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:26.480 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.478+0000 7f0ad108c640 1 --2- 172.21.15.92:0/582212692 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0acc12b490 0x7f0acc12b870 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f0ac0009830 tx=0x7f0ac002a360 comp rx=0 tx=0).stop 2024-09-21T09:43:26.480 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.478+0000 7f0ad108c640 1 -- 172.21.15.92:0/582212692 shutdown_connections 2024-09-21T09:43:26.480 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.478+0000 7f0ad108c640 1 --2- 172.21.15.92:0/582212692 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0acc094b10 0x7f0acc094f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:26.480 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.478+0000 7f0ad108c640 1 --2- 172.21.15.92:0/582212692 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0acc12bdb0 0x7f0acc0945d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:26.480 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.478+0000 7f0ad108c640 1 --2- 172.21.15.92:0/582212692 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0acc12b490 0x7f0acc12b870 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:26.480 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.478+0000 7f0ad108c640 1 -- 172.21.15.92:0/582212692 >> 172.21.15.92:0/582212692 conn(0x7f0acc09a2e0 msgr2=0x7f0acc097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:26.481 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.478+0000 7f0ad108c640 1 -- 172.21.15.92:0/582212692 shutdown_connections 2024-09-21T09:43:26.481 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.478+0000 7f0ad108c640 1 -- 172.21.15.92:0/582212692 wait complete. 2024-09-21T09:43:26.483 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.479+0000 7f0ad108c640 1 Processor -- start 2024-09-21T09:43:26.483 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.479+0000 7f0ad108c640 1 -- start start 2024-09-21T09:43:26.483 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.479+0000 7f0ad108c640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0acc094b10 0x7f0acc1c0ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:26.483 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.479+0000 7f0ad108c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0acc12b490 0x7f0acc1c0ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:26.483 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.479+0000 7f0ad108c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0acc12bdb0 0x7f0acc072430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:26.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.479+0000 7f0ad108c640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f0acc132470 con 0x7f0acc12bdb0 2024-09-21T09:43:26.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.479+0000 7f0ad108c640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f0acc0997f0 con 0x7f0acc12b490 2024-09-21T09:43:26.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.479+0000 7f0ad108c640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f0acc1325f0 con 0x7f0acc094b10 2024-09-21T09:43:26.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.480+0000 7f0aca59c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0acc12b490 0x7f0acc1c0ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:26.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.480+0000 7f0acad9d640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0acc094b10 0x7f0acc1c0ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:26.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.480+0000 7f0aca59c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0acc12b490 0x7f0acc1c0ff0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:44272/0 (socket says 172.21.15.92:44272) 2024-09-21T09:43:26.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.480+0000 7f0aca59c640 1 -- 172.21.15.92:0/410310491 learned_addr learned my addr 172.21.15.92:0/410310491 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:26.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.480+0000 7f0acb59e640 1 --2- 172.21.15.92:0/410310491 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0acc12bdb0 0x7f0acc072430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:26.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.480+0000 7f0acb59e640 1 -- 172.21.15.92:0/410310491 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0acc094b10 msgr2=0x7f0acc1c0ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:26.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.480+0000 7f0acb59e640 1 --2- 172.21.15.92:0/410310491 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0acc094b10 0x7f0acc1c0ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:26.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.480+0000 7f0acb59e640 1 -- 172.21.15.92:0/410310491 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0acc12b490 msgr2=0x7f0acc1c0ff0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:26.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.480+0000 7f0acb59e640 1 --2- 172.21.15.92:0/410310491 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0acc12b490 0x7f0acc1c0ff0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:26.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.480+0000 7f0acb59e640 1 -- 172.21.15.92:0/410310491 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0acc12c590 con 0x7f0acc12bdb0 2024-09-21T09:43:26.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.480+0000 7f0aca59c640 1 --2- 172.21.15.92:0/410310491 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0acc12b490 0x7f0acc1c0ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:26.484 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.480+0000 7f0acad9d640 1 --2- 172.21.15.92:0/410310491 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0acc094b10 0x7f0acc1c0ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:26.485 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.480+0000 7f0acb59e640 1 --2- 172.21.15.92:0/410310491 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0acc12bdb0 0x7f0acc072430 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f0ac4009fd0 tx=0x7f0ac400ef10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:26.485 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.480+0000 7f0ab37fe640 1 -- 172.21.15.92:0/410310491 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0ac400ce20 con 0x7f0acc12bdb0 2024-09-21T09:43:26.485 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.480+0000 7f0ab37fe640 1 -- 172.21.15.92:0/410310491 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f0ac4004c10 con 0x7f0acc12bdb0 2024-09-21T09:43:26.485 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.480+0000 7f0ad108c640 1 -- 172.21.15.92:0/410310491 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0acc0745b0 con 0x7f0acc12bdb0 2024-09-21T09:43:26.485 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.480+0000 7f0ab37fe640 1 -- 172.21.15.92:0/410310491 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0ac4005020 con 0x7f0acc12bdb0 2024-09-21T09:43:26.485 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.480+0000 7f0ad108c640 1 -- 172.21.15.92:0/410310491 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0acc1c1530 con 0x7f0acc12bdb0 2024-09-21T09:43:26.485 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.481+0000 7f0ab37fe640 1 -- 172.21.15.92:0/410310491 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f0ac40051c0 con 0x7f0acc12bdb0 2024-09-21T09:43:26.485 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.481+0000 7f0ad108c640 1 -- 172.21.15.92:0/410310491 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0acc12b870 con 0x7f0acc12bdb0 2024-09-21T09:43:26.485 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.482+0000 7f0ab37fe640 1 -- 172.21.15.92:0/410310491 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f0ac4069e80 con 0x7f0acc12bdb0 2024-09-21T09:43:26.487 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.485+0000 7f0ab37fe640 1 -- 172.21.15.92:0/410310491 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f0acc0745b0 con 0x7f0acc12bdb0 2024-09-21T09:43:26.665 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.663+0000 7f0ad108c640 1 -- 172.21.15.92:0/410310491 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "mgr dump", "format": "json"} v 0) -- 0x7f0acc12c800 con 0x7f0acc12bdb0 2024-09-21T09:43:26.667 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.665+0000 7f0ab37fe640 1 -- 172.21.15.92:0/410310491 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json"}]=0 v9) ==== 74+0+124219 (secure 0 0 0) 0x7f0acc12c590 con 0x7f0acc12bdb0 2024-09-21T09:43:26.668 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:26.670 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.667+0000 7f0ad108c640 1 -- 172.21.15.92:0/410310491 >> v1:172.21.15.92:6800/2299163855 conn(0x7f0ab80514e0 legacy=0x7f0ab80539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:26.670 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.667+0000 7f0ad108c640 1 -- 172.21.15.92:0/410310491 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0acc12bdb0 msgr2=0x7f0acc072430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:26.670 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.667+0000 7f0ad108c640 1 --2- 172.21.15.92:0/410310491 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0acc12bdb0 0x7f0acc072430 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f0ac4009fd0 tx=0x7f0ac400ef10 comp rx=0 tx=0).stop 2024-09-21T09:43:26.670 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.668+0000 7f0ad108c640 1 -- 172.21.15.92:0/410310491 shutdown_connections 2024-09-21T09:43:26.670 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.668+0000 7f0ad108c640 1 --2- 172.21.15.92:0/410310491 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f0acc12bdb0 0x7f0acc072430 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:26.670 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.668+0000 7f0ad108c640 1 --2- 172.21.15.92:0/410310491 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0acc12b490 0x7f0acc1c0ff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:26.670 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.668+0000 7f0ad108c640 1 --2- 172.21.15.92:0/410310491 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f0acc094b10 0x7f0acc1c0ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:26.670 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.668+0000 7f0ad108c640 1 -- 172.21.15.92:0/410310491 >> 172.21.15.92:0/410310491 conn(0x7f0acc09a2e0 msgr2=0x7f0acc091460 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:26.670 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.668+0000 7f0ad108c640 1 -- 172.21.15.92:0/410310491 shutdown_connections 2024-09-21T09:43:26.670 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.668+0000 7f0ad108c640 1 -- 172.21.15.92:0/410310491 wait complete. 2024-09-21T09:43:26.680 INFO:teuthology.orchestra.run.smithi092.stdout:{"epoch":9,"flags":0,"active_gid":14102,"active_name":"y","active_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6800","nonce":2299163855}]},"active_addr":"172.21.15.92:6800/2299163855","active_change":"2024-09-21T09:43:06.250039+0000","active_mgr_features":4540719139924082687,"available":true,"standbys":[{"gid":14113,"name":"x","mgr_features":4540719139924082687,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":21,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.92:0","nonce":1373895978}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.92:0","nonce":1689482398}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.92:0","nonce":2334132120}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.92:0","nonce":525588553}]}]} 2024-09-21T09:43:26.683 INFO:tasks.ceph.ceph_manager.ceph:mgr available! 2024-09-21T09:43:26.683 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up 2024-09-21T09:43:26.683 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:43:26.783 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.781+0000 7f2c2ced4640 1 -- 172.21.15.92:0/1500068863 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2c28095c20 msgr2=0x7f2c28096080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:26.783 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.781+0000 7f2c2ced4640 1 --2- 172.21.15.92:0/1500068863 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2c28095c20 0x7f2c28096080 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7f2c20005fd0 tx=0x7f2c2002a8e0 comp rx=0 tx=0).stop 2024-09-21T09:43:26.784 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.781+0000 7f2c2ced4640 1 -- 172.21.15.92:0/1500068863 shutdown_connections 2024-09-21T09:43:26.784 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.781+0000 7f2c2ced4640 1 --2- 172.21.15.92:0/1500068863 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2c28095c20 0x7f2c28096080 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:26.784 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.781+0000 7f2c2ced4640 1 --2- 172.21.15.92:0/1500068863 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2c2812c160 0x7f2c280956e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:26.784 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.781+0000 7f2c2ced4640 1 --2- 172.21.15.92:0/1500068863 >> v2:172.21.15.92:3300/0 conn(0x7f2c280991f0 0x7f2c2812bc20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:26.784 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.781+0000 7f2c2ced4640 1 -- 172.21.15.92:0/1500068863 >> 172.21.15.92:0/1500068863 conn(0x7f2c2809a2e0 msgr2=0x7f2c28097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:26.784 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.781+0000 7f2c2ced4640 1 -- 172.21.15.92:0/1500068863 shutdown_connections 2024-09-21T09:43:26.784 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.781+0000 7f2c2ced4640 1 -- 172.21.15.92:0/1500068863 wait complete. 2024-09-21T09:43:26.786 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.782+0000 7f2c2ced4640 1 Processor -- start 2024-09-21T09:43:26.786 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.782+0000 7f2c2ced4640 1 -- start start 2024-09-21T09:43:26.787 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.783+0000 7f2c2ced4640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2c28095c20 0x7f2c281c2d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:26.787 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.783+0000 7f2c2ced4640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2c280991f0 0x7f2c281c3260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:26.787 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.783+0000 7f2c2ced4640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2c2812c160 0x7f2c2812e690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:26.787 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.783+0000 7f2c2ced4640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f2c28094fe0 con 0x7f2c28095c20 2024-09-21T09:43:26.787 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.783+0000 7f2c2ced4640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f2c28094e60 con 0x7f2c280991f0 2024-09-21T09:43:26.787 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.783+0000 7f2c2ced4640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f2c28095160 con 0x7f2c2812c160 2024-09-21T09:43:26.787 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.783+0000 7f2c2659c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2c28095c20 0x7f2c281c2d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:26.787 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.783+0000 7f2c25d9b640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2c280991f0 0x7f2c281c3260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:26.787 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.783+0000 7f2c2659c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2c28095c20 0x7f2c281c2d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:42672/0 (socket says 172.21.15.92:42672) 2024-09-21T09:43:26.787 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.783+0000 7f2c26d9d640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2c2812c160 0x7f2c2812e690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:26.787 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.783+0000 7f2c2659c640 1 -- 172.21.15.92:0/2641820674 learned_addr learned my addr 172.21.15.92:0/2641820674 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:26.787 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.783+0000 7f2c25d9b640 1 -- 172.21.15.92:0/2641820674 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2c2812c160 msgr2=0x7f2c2812e690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:26.787 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.783+0000 7f2c25d9b640 1 --2- 172.21.15.92:0/2641820674 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2c2812c160 0x7f2c2812e690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:26.787 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.783+0000 7f2c25d9b640 1 -- 172.21.15.92:0/2641820674 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2c28095c20 msgr2=0x7f2c281c2d20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:26.788 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.783+0000 7f2c25d9b640 1 --2- 172.21.15.92:0/2641820674 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2c28095c20 0x7f2c281c2d20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:26.788 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.783+0000 7f2c25d9b640 1 -- 172.21.15.92:0/2641820674 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2c28130700 con 0x7f2c280991f0 2024-09-21T09:43:26.788 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.783+0000 7f2c2659c640 1 --2- 172.21.15.92:0/2641820674 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2c28095c20 0x7f2c281c2d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:26.788 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.783+0000 7f2c25d9b640 1 --2- 172.21.15.92:0/2641820674 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2c280991f0 0x7f2c281c3260 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7f2c1c00e9f0 tx=0x7f2c1c00eeb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:26.788 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.783+0000 7f2c16ffd640 1 -- 172.21.15.92:0/2641820674 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2c1c00cc60 con 0x7f2c280991f0 2024-09-21T09:43:26.788 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.783+0000 7f2c16ffd640 1 -- 172.21.15.92:0/2641820674 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2c1c004020 con 0x7f2c280991f0 2024-09-21T09:43:26.788 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.783+0000 7f2c16ffd640 1 -- 172.21.15.92:0/2641820674 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2c1c004470 con 0x7f2c280991f0 2024-09-21T09:43:26.788 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.783+0000 7f2c2ced4640 1 -- 172.21.15.92:0/2641820674 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2c28094030 con 0x7f2c280991f0 2024-09-21T09:43:26.788 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.784+0000 7f2c2ced4640 1 -- 172.21.15.92:0/2641820674 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f2c281c37a0 con 0x7f2c280991f0 2024-09-21T09:43:26.788 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.785+0000 7f2c16ffd640 1 -- 172.21.15.92:0/2641820674 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f2c1c005020 con 0x7f2c280991f0 2024-09-21T09:43:26.788 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.785+0000 7f2c2ced4640 1 -- 172.21.15.92:0/2641820674 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2c2809cec0 con 0x7f2c280991f0 2024-09-21T09:43:26.788 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.785+0000 7f2c16ffd640 1 -- 172.21.15.92:0/2641820674 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f2c1c0695d0 con 0x7f2c280991f0 2024-09-21T09:43:26.790 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.788+0000 7f2c16ffd640 1 -- 172.21.15.92:0/2641820674 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f2c2809cec0 con 0x7f2c280991f0 2024-09-21T09:43:26.938 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.936+0000 7f2c2ced4640 1 -- 172.21.15.92:0/2641820674 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f2c28096080 con 0x7f2c280991f0 2024-09-21T09:43:26.939 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.937+0000 7f2c16ffd640 1 -- 172.21.15.92:0/2641820674 <== mon.1 v2:172.21.15.92:3301/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v33) ==== 74+0+9145 (secure 0 0 0) 0x7f2c28094030 con 0x7f2c280991f0 2024-09-21T09:43:26.939 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:26.939 INFO:teuthology.orchestra.run.smithi092.stdout:{"epoch":33,"fsid":"5f250e6c-4e36-49b1-ab62-d9454369fa03","created":"2024-09-21T09:41:34.680338+0000","modified":"2024-09-21T09:43:26.338713+0000","last_up_change":"2024-09-21T09:43:20.303761+0000","last_in_change":"2024-09-21T09:41:37.598778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":9,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"squid","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:41:50.040313+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"32","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:41:52.054798+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f3314f59-3374-4072-85c9-f8c4e458efc8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":21,"up_from":27,"up_thru":27,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6805","nonce":7087154}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6806","nonce":7087154}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6808","nonce":7087154}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6807","nonce":7087154}]},"public_addr":"172.21.15.92:6805/7087154","cluster_addr":"172.21.15.92:6806/7087154","heartbeat_back_addr":"172.21.15.92:6808/7087154","heartbeat_front_addr":"172.21.15.92:6807/7087154","state":["exists","up"]},{"osd":1,"uuid":"cf0486af-ba68-4763-a364-c1f2181604a7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":21,"up_from":28,"up_thru":28,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6801","nonce":2577383826}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6802","nonce":2577383826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6804","nonce":2577383826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6803","nonce":2577383826}]},"public_addr":"172.21.15.92:6801/2577383826","cluster_addr":"172.21.15.92:6802/2577383826","heartbeat_back_addr":"172.21.15.92:6804/2577383826","heartbeat_front_addr":"172.21.15.92:6803/2577383826","state":["exists","up"]},{"osd":2,"uuid":"5c587421-3409-4469-a2ee-b2ea20cd45e5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":23,"up_from":29,"up_thru":30,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6809","nonce":4235634095}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6810","nonce":4235634095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6812","nonce":4235634095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6811","nonce":4235634095}]},"public_addr":"172.21.15.92:6809/4235634095","cluster_addr":"172.21.15.92:6810/4235634095","heartbeat_back_addr":"172.21.15.92:6812/4235634095","heartbeat_front_addr":"172.21.15.92:6811/4235634095","state":["exists","up"]},{"osd":3,"uuid":"b98663b7-de4d-4c12-ac91-8c33f159da35","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":24,"up_from":30,"up_thru":30,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6813","nonce":138232475}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6814","nonce":138232475}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6816","nonce":138232475}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6815","nonce":138232475}]},"public_addr":"172.21.15.92:6813/138232475","cluster_addr":"172.21.15.92:6814/138232475","heartbeat_back_addr":"172.21.15.92:6816/138232475","heartbeat_front_addr":"172.21.15.92:6815/138232475","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.934317+0000","dead_epoch":22},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:45.144020+0000","dead_epoch":23},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.895848+0000","dead_epoch":24},{"osd":3,"down_stamp":"2024-09-21T09:43:13.277270+0000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.636219+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.92:0/230938560":"2024-09-22T09:43:06.249869+0000","172.21.15.92:0/162460762":"2024-09-22T09:43:06.249869+0000","172.21.15.92:6800/1280404317":"2024-09-22T09:43:06.249869+0000","172.21.15.92:0/3587832392":"2024-09-22T09:43:06.249869+0000","172.21.15.92:0/2147030555":"2024-09-22T09:43:06.249869+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:43:26.941 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.939+0000 7f2c2ced4640 1 -- 172.21.15.92:0/2641820674 >> v1:172.21.15.92:6800/2299163855 conn(0x7f2c080514b0 legacy=0x7f2c08053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:26.941 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.939+0000 7f2c2ced4640 1 -- 172.21.15.92:0/2641820674 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2c280991f0 msgr2=0x7f2c281c3260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:26.941 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.939+0000 7f2c2ced4640 1 --2- 172.21.15.92:0/2641820674 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2c280991f0 0x7f2c281c3260 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7f2c1c00e9f0 tx=0x7f2c1c00eeb0 comp rx=0 tx=0).stop 2024-09-21T09:43:26.941 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.939+0000 7f2c2ced4640 1 -- 172.21.15.92:0/2641820674 shutdown_connections 2024-09-21T09:43:26.941 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.939+0000 7f2c2ced4640 1 --2- 172.21.15.92:0/2641820674 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2c2812c160 0x7f2c2812e690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:26.941 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.939+0000 7f2c2ced4640 1 --2- 172.21.15.92:0/2641820674 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2c280991f0 0x7f2c281c3260 unknown :-1 s=CLOSED pgs=34 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:26.941 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.939+0000 7f2c2ced4640 1 --2- 172.21.15.92:0/2641820674 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2c28095c20 0x7f2c281c2d20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:26.941 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.939+0000 7f2c2ced4640 1 -- 172.21.15.92:0/2641820674 >> 172.21.15.92:0/2641820674 conn(0x7f2c2809a2e0 msgr2=0x7f2c28091c60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:26.941 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.939+0000 7f2c2ced4640 1 -- 172.21.15.92:0/2641820674 shutdown_connections 2024-09-21T09:43:26.941 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:26.939+0000 7f2c2ced4640 1 -- 172.21.15.92:0/2641820674 wait complete. 2024-09-21T09:43:26.952 INFO:tasks.ceph.ceph_manager.ceph:all up! 2024-09-21T09:43:26.952 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:43:27.052 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.049+0000 7fd73bc8c640 1 -- 172.21.15.92:0/1008380033 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd73412ddf0 msgr2=0x7fd7341301e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.052 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.049+0000 7fd73bc8c640 1 --2- 172.21.15.92:0/1008380033 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd73412ddf0 0x7fd7341301e0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7fd724009830 tx=0x7fd72402a3f0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.052 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.049+0000 7fd73bc8c640 1 -- 172.21.15.92:0/1008380033 shutdown_connections 2024-09-21T09:43:27.052 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.049+0000 7fd73bc8c640 1 --2- 172.21.15.92:0/1008380033 >> v2:172.21.15.92:3300/0 conn(0x7fd734130720 0x7fd734132b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.052 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.049+0000 7fd73bc8c640 1 --2- 172.21.15.92:0/1008380033 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd73412ddf0 0x7fd7341301e0 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.052 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.049+0000 7fd73bc8c640 1 -- 172.21.15.92:0/1008380033 >> 172.21.15.92:0/1008380033 conn(0x7fd73409a2e0 msgr2=0x7fd734097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:27.053 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.050+0000 7fd73bc8c640 1 -- 172.21.15.92:0/1008380033 shutdown_connections 2024-09-21T09:43:27.053 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.050+0000 7fd73bc8c640 1 -- 172.21.15.92:0/1008380033 wait complete. 2024-09-21T09:43:27.053 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.050+0000 7fd73bc8c640 1 Processor -- start 2024-09-21T09:43:27.053 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.050+0000 7fd73bc8c640 1 -- start start 2024-09-21T09:43:27.055 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.051+0000 7fd73bc8c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd73412af70 0x7fd7341c4890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.055 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.051+0000 7fd73bc8c640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd73412ddf0 0x7fd7341c4dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.055 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.051+0000 7fd73bc8c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd734130720 0x7fd7341c5310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.055 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.051+0000 7fd73bc8c640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fd734133550 con 0x7fd73412af70 2024-09-21T09:43:27.055 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.051+0000 7fd73bc8c640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fd7341333d0 con 0x7fd734130720 2024-09-21T09:43:27.055 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.051+0000 7fd73bc8c640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fd7341336d0 con 0x7fd73412ddf0 2024-09-21T09:43:27.055 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.051+0000 7fd739227640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd73412ddf0 0x7fd7341c4dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.055 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.051+0000 7fd739a28640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd73412af70 0x7fd7341c4890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.055 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.051+0000 7fd739227640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd73412ddf0 0x7fd7341c4dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:48110/0 (socket says 172.21.15.92:48110) 2024-09-21T09:43:27.056 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.051+0000 7fd739227640 1 -- 172.21.15.92:0/2958089071 learned_addr learned my addr 172.21.15.92:0/2958089071 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:27.056 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.051+0000 7fd73a229640 1 --2- 172.21.15.92:0/2958089071 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd734130720 0x7fd7341c5310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.056 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.051+0000 7fd739a28640 1 -- 172.21.15.92:0/2958089071 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd73412ddf0 msgr2=0x7fd7341c4dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.056 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.051+0000 7fd739a28640 1 --2- 172.21.15.92:0/2958089071 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd73412ddf0 0x7fd7341c4dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.056 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.051+0000 7fd739a28640 1 -- 172.21.15.92:0/2958089071 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd734130720 msgr2=0x7fd7341c5310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.056 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.051+0000 7fd739a28640 1 --2- 172.21.15.92:0/2958089071 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd734130720 0x7fd7341c5310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.056 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.051+0000 7fd739a28640 1 -- 172.21.15.92:0/2958089071 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd734129c90 con 0x7fd73412af70 2024-09-21T09:43:27.056 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.051+0000 7fd739227640 1 --2- 172.21.15.92:0/2958089071 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd73412ddf0 0x7fd7341c4dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:27.056 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.051+0000 7fd739a28640 1 --2- 172.21.15.92:0/2958089071 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd73412af70 0x7fd7341c4890 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7fd73000c430 tx=0x7fd73000c8f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:27.056 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.051+0000 7fd72a7fc640 1 -- 172.21.15.92:0/2958089071 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd730017070 con 0x7fd73412af70 2024-09-21T09:43:27.056 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.051+0000 7fd72a7fc640 1 -- 172.21.15.92:0/2958089071 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fd7300053e0 con 0x7fd73412af70 2024-09-21T09:43:27.056 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.052+0000 7fd73bc8c640 1 -- 172.21.15.92:0/2958089071 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd734090600 con 0x7fd73412af70 2024-09-21T09:43:27.056 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.052+0000 7fd72a7fc640 1 -- 172.21.15.92:0/2958089071 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd730003850 con 0x7fd73412af70 2024-09-21T09:43:27.057 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.052+0000 7fd73bc8c640 1 -- 172.21.15.92:0/2958089071 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd73412a670 con 0x7fd73412af70 2024-09-21T09:43:27.057 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.053+0000 7fd73bc8c640 1 -- 172.21.15.92:0/2958089071 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd734133c60 con 0x7fd73412af70 2024-09-21T09:43:27.057 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.053+0000 7fd72a7fc640 1 -- 172.21.15.92:0/2958089071 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7fd73001a070 con 0x7fd73412af70 2024-09-21T09:43:27.057 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.053+0000 7fd72a7fc640 1 -- 172.21.15.92:0/2958089071 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fd730068340 con 0x7fd73412af70 2024-09-21T09:43:27.058 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.057+0000 7fd72a7fc640 1 -- 172.21.15.92:0/2958089071 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fd7340967f0 con 0x7fd73412af70 2024-09-21T09:43:27.210 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.208+0000 7fd73bc8c640 1 -- 172.21.15.92:0/2958089071 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fd7340699c0 con 0x7fd73412af70 2024-09-21T09:43:27.210 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.208+0000 7fd72a7fc640 1 -- 172.21.15.92:0/2958089071 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v33) ==== 74+0+9145 (secure 0 0 0) 0x7fd7300673f0 con 0x7fd73412af70 2024-09-21T09:43:27.210 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:27.211 INFO:teuthology.orchestra.run.smithi092.stdout:{"epoch":33,"fsid":"5f250e6c-4e36-49b1-ab62-d9454369fa03","created":"2024-09-21T09:41:34.680338+0000","modified":"2024-09-21T09:43:26.338713+0000","last_up_change":"2024-09-21T09:43:20.303761+0000","last_in_change":"2024-09-21T09:41:37.598778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":9,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"squid","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:41:50.040313+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"32","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:41:52.054798+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f3314f59-3374-4072-85c9-f8c4e458efc8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":21,"up_from":27,"up_thru":27,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6805","nonce":7087154}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6806","nonce":7087154}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6808","nonce":7087154}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6807","nonce":7087154}]},"public_addr":"172.21.15.92:6805/7087154","cluster_addr":"172.21.15.92:6806/7087154","heartbeat_back_addr":"172.21.15.92:6808/7087154","heartbeat_front_addr":"172.21.15.92:6807/7087154","state":["exists","up"]},{"osd":1,"uuid":"cf0486af-ba68-4763-a364-c1f2181604a7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":21,"up_from":28,"up_thru":28,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6801","nonce":2577383826}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6802","nonce":2577383826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6804","nonce":2577383826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6803","nonce":2577383826}]},"public_addr":"172.21.15.92:6801/2577383826","cluster_addr":"172.21.15.92:6802/2577383826","heartbeat_back_addr":"172.21.15.92:6804/2577383826","heartbeat_front_addr":"172.21.15.92:6803/2577383826","state":["exists","up"]},{"osd":2,"uuid":"5c587421-3409-4469-a2ee-b2ea20cd45e5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":23,"up_from":29,"up_thru":30,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6809","nonce":4235634095}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6810","nonce":4235634095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6812","nonce":4235634095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6811","nonce":4235634095}]},"public_addr":"172.21.15.92:6809/4235634095","cluster_addr":"172.21.15.92:6810/4235634095","heartbeat_back_addr":"172.21.15.92:6812/4235634095","heartbeat_front_addr":"172.21.15.92:6811/4235634095","state":["exists","up"]},{"osd":3,"uuid":"b98663b7-de4d-4c12-ac91-8c33f159da35","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":24,"up_from":30,"up_thru":30,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6813","nonce":138232475}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6814","nonce":138232475}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6816","nonce":138232475}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6815","nonce":138232475}]},"public_addr":"172.21.15.92:6813/138232475","cluster_addr":"172.21.15.92:6814/138232475","heartbeat_back_addr":"172.21.15.92:6816/138232475","heartbeat_front_addr":"172.21.15.92:6815/138232475","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.934317+0000","dead_epoch":22},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:45.144020+0000","dead_epoch":23},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.895848+0000","dead_epoch":24},{"osd":3,"down_stamp":"2024-09-21T09:43:13.277270+0000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.636219+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.92:0/230938560":"2024-09-22T09:43:06.249869+0000","172.21.15.92:0/162460762":"2024-09-22T09:43:06.249869+0000","172.21.15.92:6800/1280404317":"2024-09-22T09:43:06.249869+0000","172.21.15.92:0/3587832392":"2024-09-22T09:43:06.249869+0000","172.21.15.92:0/2147030555":"2024-09-22T09:43:06.249869+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:43:27.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.210+0000 7fd73bc8c640 1 -- 172.21.15.92:0/2958089071 >> v1:172.21.15.92:6800/2299163855 conn(0x7fd7180514b0 legacy=0x7fd718053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.210+0000 7fd73bc8c640 1 -- 172.21.15.92:0/2958089071 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd73412af70 msgr2=0x7fd7341c4890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.212 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.210+0000 7fd73bc8c640 1 --2- 172.21.15.92:0/2958089071 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd73412af70 0x7fd7341c4890 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7fd73000c430 tx=0x7fd73000c8f0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.210+0000 7fd73bc8c640 1 -- 172.21.15.92:0/2958089071 shutdown_connections 2024-09-21T09:43:27.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.210+0000 7fd73bc8c640 1 --2- 172.21.15.92:0/2958089071 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd734130720 0x7fd7341c5310 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.210+0000 7fd73bc8c640 1 --2- 172.21.15.92:0/2958089071 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd73412ddf0 0x7fd7341c4dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.210+0000 7fd73bc8c640 1 --2- 172.21.15.92:0/2958089071 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd73412af70 0x7fd7341c4890 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.210+0000 7fd73bc8c640 1 -- 172.21.15.92:0/2958089071 >> 172.21.15.92:0/2958089071 conn(0x7fd73409a2e0 msgr2=0x7fd73412edc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:27.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.210+0000 7fd73bc8c640 1 -- 172.21.15.92:0/2958089071 shutdown_connections 2024-09-21T09:43:27.213 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.210+0000 7fd73bc8c640 1 -- 172.21.15.92:0/2958089071 wait complete. 2024-09-21T09:43:27.223 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 flush_pg_stats 2024-09-21T09:43:27.224 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 flush_pg_stats 2024-09-21T09:43:27.224 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 flush_pg_stats 2024-09-21T09:43:27.225 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 flush_pg_stats 2024-09-21T09:43:27.332 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.330+0000 7f7af12a5640 1 -- 172.21.15.92:0/3632377746 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7aec0711d0 msgr2=0x7f7aec0709d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.332 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.330+0000 7f7af12a5640 1 --2- 172.21.15.92:0/3632377746 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7aec0711d0 0x7f7aec0709d0 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f7ae0009830 tx=0x7f7ae002a3f0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.333 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.330+0000 7fcc74007640 1 -- 172.21.15.92:0/2962395910 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcc6c095f70 msgr2=0x7fcc6c096350 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.333 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.330+0000 7fcc74007640 1 --2- 172.21.15.92:0/2962395910 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcc6c095f70 0x7fcc6c096350 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fcc68005fd0 tx=0x7fcc6802a540 comp rx=0 tx=0).stop 2024-09-21T09:43:27.333 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.330+0000 7fcc74007640 1 -- 172.21.15.92:0/2962395910 shutdown_connections 2024-09-21T09:43:27.333 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.330+0000 7fcc74007640 1 --2- 172.21.15.92:0/2962395910 >> v2:172.21.15.92:3300/0 conn(0x7fcc6c096890 0x7fcc6c1271e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.333 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.330+0000 7fcc74007640 1 --2- 172.21.15.92:0/2962395910 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcc6c095f70 0x7fcc6c096350 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.333 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.330+0000 7fcc74007640 1 -- 172.21.15.92:0/2962395910 >> 172.21.15.92:0/2962395910 conn(0x7fcc6c09afd0 msgr2=0x7fcc6c09b3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:27.333 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.330+0000 7f7af12a5640 1 -- 172.21.15.92:0/3632377746 shutdown_connections 2024-09-21T09:43:27.333 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.330+0000 7f7af12a5640 1 --2- 172.21.15.92:0/3632377746 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7aec0711d0 0x7f7aec0709d0 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.333 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.330+0000 7f7af12a5640 1 --2- 172.21.15.92:0/3632377746 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7aec0700b0 0x7f7aec070490 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.334 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.330+0000 7f7af12a5640 1 -- 172.21.15.92:0/3632377746 >> 172.21.15.92:0/3632377746 conn(0x7f7aec09b9a0 msgr2=0x7f7aec09bd80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:27.334 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.330+0000 7f0468a70640 1 -- 172.21.15.92:0/2132386735 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0464128970 msgr2=0x7f0464133b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.334 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.330+0000 7f0468a70640 1 --2- 172.21.15.92:0/2132386735 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0464128970 0x7f0464133b60 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f045c009be0 tx=0x7f045c02a800 comp rx=0 tx=0).stop 2024-09-21T09:43:27.334 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.330+0000 7f7af12a5640 1 -- 172.21.15.92:0/3632377746 shutdown_connections 2024-09-21T09:43:27.335 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.331+0000 7fcc74007640 1 -- 172.21.15.92:0/2962395910 shutdown_connections 2024-09-21T09:43:27.335 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.331+0000 7f7af12a5640 1 -- 172.21.15.92:0/3632377746 wait complete. 2024-09-21T09:43:27.335 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.331+0000 7fcc74007640 1 -- 172.21.15.92:0/2962395910 wait complete. 2024-09-21T09:43:27.336 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.331+0000 7f0468a70640 1 -- 172.21.15.92:0/2132386735 shutdown_connections 2024-09-21T09:43:27.336 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.331+0000 7f0468a70640 1 --2- 172.21.15.92:0/2132386735 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0464128970 0x7f0464133b60 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.336 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.331+0000 7f0468a70640 1 --2- 172.21.15.92:0/2132386735 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f046412bad0 0x7f046412beb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.336 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.331+0000 7f0468a70640 1 -- 172.21.15.92:0/2132386735 >> 172.21.15.92:0/2132386735 conn(0x7f046409afd0 msgr2=0x7f046409b3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:27.336 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.331+0000 7f0468a70640 1 -- 172.21.15.92:0/2132386735 shutdown_connections 2024-09-21T09:43:27.336 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.331+0000 7f0468a70640 1 -- 172.21.15.92:0/2132386735 wait complete. 2024-09-21T09:43:27.337 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.331+0000 7f44acfa5640 1 -- 172.21.15.92:0/1472426302 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f44a8098bc0 msgr2=0x7f44a8098fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.337 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.331+0000 7f44acfa5640 1 --2- 172.21.15.92:0/1472426302 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f44a8098bc0 0x7f44a8098fa0 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f4498009830 tx=0x7f449802a360 comp rx=0 tx=0).stop 2024-09-21T09:43:27.337 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.331+0000 7f44acfa5640 1 -- 172.21.15.92:0/1472426302 shutdown_connections 2024-09-21T09:43:27.337 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.331+0000 7f44acfa5640 1 --2- 172.21.15.92:0/1472426302 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f44a80994e0 0x7f44a812e980 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.338 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.331+0000 7f44acfa5640 1 --2- 172.21.15.92:0/1472426302 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f44a8098bc0 0x7f44a8098fa0 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.338 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.331+0000 7f44acfa5640 1 -- 172.21.15.92:0/1472426302 >> 172.21.15.92:0/1472426302 conn(0x7f44a80a1630 msgr2=0x7f44a80a1a40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:27.338 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.331+0000 7f44acfa5640 1 -- 172.21.15.92:0/1472426302 shutdown_connections 2024-09-21T09:43:27.338 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.331+0000 7f44acfa5640 1 -- 172.21.15.92:0/1472426302 wait complete. 2024-09-21T09:43:27.338 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.332+0000 7fcc74007640 1 Processor -- start 2024-09-21T09:43:27.338 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.332+0000 7f0468a70640 1 Processor -- start 2024-09-21T09:43:27.339 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.332+0000 7fcc74007640 1 -- start start 2024-09-21T09:43:27.339 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.332+0000 7f0468a70640 1 -- start start 2024-09-21T09:43:27.339 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.332+0000 7f7af12a5640 1 Processor -- start 2024-09-21T09:43:27.339 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.332+0000 7f7af12a5640 1 -- start start 2024-09-21T09:43:27.340 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f0468a70640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0464128970 0x7f046406e890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.340 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f0468a70640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f046412bad0 0x7f046406edd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.340 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f0468a70640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f046412c3f0 0x7f046406f310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.340 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7fcc74007640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcc6c095f70 0x7fcc6c122190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.341 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f0462d9d640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f046412c3f0 0x7f046406f310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.341 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f046259c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0464128970 0x7f046406e890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.341 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f0462d9d640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f046412c3f0 0x7f046406f310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:48144/0 (socket says 172.21.15.92:48144) 2024-09-21T09:43:27.341 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f0461d9b640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f046412bad0 0x7f046406edd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.341 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f046259c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0464128970 0x7f046406e890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:44330/0 (socket says 172.21.15.92:44330) 2024-09-21T09:43:27.341 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f0462d9d640 1 -- 172.21.15.92:0/1229720013 learned_addr learned my addr 172.21.15.92:0/1229720013 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:27.341 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f0461d9b640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f046412bad0 0x7f046406edd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:42744/0 (socket says 172.21.15.92:42744) 2024-09-21T09:43:27.341 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f44acfa5640 1 Processor -- start 2024-09-21T09:43:27.342 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f0468a70640 1 -- 172.21.15.92:0/1229720013 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f04641346d0 con 0x7f046412bad0 2024-09-21T09:43:27.342 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f7af12a5640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7aec0700b0 0x7f7aec123320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.342 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f0468a70640 1 -- 172.21.15.92:0/1229720013 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f0464134550 con 0x7f0464128970 2024-09-21T09:43:27.342 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f0468a70640 1 -- 172.21.15.92:0/1229720013 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f0464134850 con 0x7f046412c3f0 2024-09-21T09:43:27.343 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f7af12a5640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7aec0711d0 0x7f7aec123860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.343 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f44acfa5640 1 -- start start 2024-09-21T09:43:27.343 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7fcc71da3640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcc6c095f70 0x7fcc6c122190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.343 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7fcc71da3640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcc6c095f70 0x7fcc6c122190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:44342/0 (socket says 172.21.15.92:44342) 2024-09-21T09:43:27.343 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7fcc71da3640 1 -- 172.21.15.92:0/3770039411 learned_addr learned my addr 172.21.15.92:0/3770039411 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:27.344 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f7aeaffd640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7aec0700b0 0x7f7aec123320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.344 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f7af12a5640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7aec12bdd0 0x7f7aec123da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.344 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f7aeaffd640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7aec0700b0 0x7f7aec123320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:48160/0 (socket says 172.21.15.92:48160) 2024-09-21T09:43:27.344 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7fcc74007640 1 --2- 172.21.15.92:0/3770039411 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcc6c096890 0x7fcc6c1226d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.345 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f7aea7fc640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7aec0711d0 0x7f7aec123860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.345 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f7aeaffd640 1 -- 172.21.15.92:0/71156227 learned_addr learned my addr 172.21.15.92:0/71156227 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:27.345 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f7af12a5640 1 -- 172.21.15.92:0/71156227 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f7aec06f8d0 con 0x7f7aec0711d0 2024-09-21T09:43:27.345 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f7af12a5640 1 -- 172.21.15.92:0/71156227 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f7aec06f750 con 0x7f7aec12bdd0 2024-09-21T09:43:27.345 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f7af12a5640 1 -- 172.21.15.92:0/71156227 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f7aec06fa50 con 0x7f7aec0700b0 2024-09-21T09:43:27.345 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f7aea7fc640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7aec0711d0 0x7f7aec123860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:42752/0 (socket says 172.21.15.92:42752) 2024-09-21T09:43:27.346 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7f7aeb7fe640 1 --2- 172.21.15.92:0/71156227 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7aec12bdd0 0x7f7aec123da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.346 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.333+0000 7fcc74007640 1 --2- 172.21.15.92:0/3770039411 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcc6c127720 0x7fcc6c122c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.346 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7fcc715a2640 1 --2- 172.21.15.92:0/3770039411 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcc6c096890 0x7fcc6c1226d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.346 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f7aeaffd640 1 -- 172.21.15.92:0/71156227 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7aec12bdd0 msgr2=0x7f7aec123da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.347 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f7aeaffd640 1 --2- 172.21.15.92:0/71156227 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7aec12bdd0 0x7f7aec123da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.347 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f7aeaffd640 1 -- 172.21.15.92:0/71156227 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7aec0711d0 msgr2=0x7f7aec123860 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.347 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f7aeaffd640 1 --2- 172.21.15.92:0/71156227 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7aec0711d0 0x7f7aec123860 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.347 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f7aeaffd640 1 -- 172.21.15.92:0/71156227 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7aec0734c0 con 0x7f7aec0700b0 2024-09-21T09:43:27.347 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f0461d9b640 1 -- 172.21.15.92:0/1229720013 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f046412c3f0 msgr2=0x7f046406f310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.347 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f0461d9b640 1 --2- 172.21.15.92:0/1229720013 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f046412c3f0 0x7f046406f310 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.347 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f0461d9b640 1 -- 172.21.15.92:0/1229720013 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0464128970 msgr2=0x7f046406e890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.347 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f0461d9b640 1 --2- 172.21.15.92:0/1229720013 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0464128970 0x7f046406e890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.347 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f0461d9b640 1 -- 172.21.15.92:0/1229720013 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0464070810 con 0x7f046412bad0 2024-09-21T09:43:27.347 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f046259c640 1 --2- 172.21.15.92:0/1229720013 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f0464128970 0x7f046406e890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:27.348 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f7aeb7fe640 1 --2- 172.21.15.92:0/71156227 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7aec12bdd0 0x7f7aec123da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:43:27.348 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f7aeaffd640 1 --2- 172.21.15.92:0/71156227 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7aec0700b0 0x7f7aec123320 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f7adc00e9c0 tx=0x7f7adc00ee80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:27.348 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f0461d9b640 1 --2- 172.21.15.92:0/1229720013 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f046412bad0 0x7f046406edd0 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f045800c400 tx=0x7f045800c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:27.348 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7fcc725a4640 1 --2- 172.21.15.92:0/3770039411 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcc6c127720 0x7fcc6c122c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.349 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f0452ffd640 1 -- 172.21.15.92:0/1229720013 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0458011790 con 0x7f046412bad0 2024-09-21T09:43:27.349 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f7ad37fe640 1 -- 172.21.15.92:0/71156227 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7adc00ccc0 con 0x7f7aec0700b0 2024-09-21T09:43:27.349 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f7ad37fe640 1 -- 172.21.15.92:0/71156227 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f7adc005020 con 0x7f7aec0700b0 2024-09-21T09:43:27.349 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f0452ffd640 1 -- 172.21.15.92:0/1229720013 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f0458017070 con 0x7f046412bad0 2024-09-21T09:43:27.349 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f0468a70640 1 -- 172.21.15.92:0/1229720013 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f046406ff30 con 0x7f046412bad0 2024-09-21T09:43:27.350 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f7af12a5640 1 -- 172.21.15.92:0/71156227 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7aec129820 con 0x7f7aec0700b0 2024-09-21T09:43:27.350 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f0452ffd640 1 -- 172.21.15.92:0/1229720013 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0458011d00 con 0x7f046412bad0 2024-09-21T09:43:27.350 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f0468a70640 1 -- 172.21.15.92:0/1229720013 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0464071320 con 0x7f046412bad0 2024-09-21T09:43:27.351 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7fcc74007640 1 -- 172.21.15.92:0/3770039411 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fcc6c095070 con 0x7fcc6c127720 2024-09-21T09:43:27.351 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7fcc74007640 1 -- 172.21.15.92:0/3770039411 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fcc6c128070 con 0x7fcc6c095f70 2024-09-21T09:43:27.351 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7fcc74007640 1 -- 172.21.15.92:0/3770039411 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fcc6c0951f0 con 0x7fcc6c096890 2024-09-21T09:43:27.351 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.335+0000 7f44acfa5640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f44a8098bc0 0x7f44a8129770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.352 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.334+0000 7f7ad37fe640 1 -- 172.21.15.92:0/71156227 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7adc00ce60 con 0x7f7aec0700b0 2024-09-21T09:43:27.352 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.335+0000 7f7af12a5640 1 -- 172.21.15.92:0/71156227 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f7aec125ef0 con 0x7f7aec0700b0 2024-09-21T09:43:27.352 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.335+0000 7f44acfa5640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f44a80994e0 0x7f44a8129cb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.352 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.335+0000 7f44a659c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f44a8098bc0 0x7f44a8129770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.352 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.335+0000 7f44a659c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f44a8098bc0 0x7f44a8129770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:42766/0 (socket says 172.21.15.92:42766) 2024-09-21T09:43:27.352 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.335+0000 7f44acfa5640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f44a812eec0 0x7f44a812a1f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.353 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.335+0000 7fcc71da3640 1 -- 172.21.15.92:0/3770039411 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcc6c096890 msgr2=0x7fcc6c1226d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.353 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.335+0000 7fcc71da3640 1 --2- 172.21.15.92:0/3770039411 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fcc6c096890 0x7fcc6c1226d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.353 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.335+0000 7fcc71da3640 1 -- 172.21.15.92:0/3770039411 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcc6c127720 msgr2=0x7fcc6c122c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.353 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.335+0000 7fcc71da3640 1 --2- 172.21.15.92:0/3770039411 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcc6c127720 0x7fcc6c122c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.353 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.335+0000 7fcc71da3640 1 -- 172.21.15.92:0/3770039411 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcc6c1248b0 con 0x7fcc6c095f70 2024-09-21T09:43:27.353 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.335+0000 7fcc725a4640 1 --2- 172.21.15.92:0/3770039411 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fcc6c127720 0x7fcc6c122c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:27.354 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.335+0000 7f44acfa5640 1 -- 172.21.15.92:0/2141194680 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f44a8137990 con 0x7f44a8098bc0 2024-09-21T09:43:27.354 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.336+0000 7f0468a70640 1 -- 172.21.15.92:0/1229720013 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f0464134de0 con 0x7f046412bad0 2024-09-21T09:43:27.355 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.335+0000 7f44acfa5640 1 -- 172.21.15.92:0/2141194680 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f44a8137810 con 0x7f44a812eec0 2024-09-21T09:43:27.355 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.335+0000 7f44acfa5640 1 -- 172.21.15.92:0/2141194680 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f44a8137b10 con 0x7f44a80994e0 2024-09-21T09:43:27.355 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.335+0000 7f44a5d9b640 1 --2- 172.21.15.92:0/2141194680 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f44a80994e0 0x7f44a8129cb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.355 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.335+0000 7f44a6d9d640 1 --2- 172.21.15.92:0/2141194680 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f44a812eec0 0x7f44a812a1f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.355 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.335+0000 7f44a659c640 1 -- 172.21.15.92:0/2141194680 learned_addr learned my addr 172.21.15.92:0/2141194680 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:27.355 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.336+0000 7f44a5d9b640 1 -- 172.21.15.92:0/2141194680 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f44a812eec0 msgr2=0x7f44a812a1f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.355 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.336+0000 7f0452ffd640 1 -- 172.21.15.92:0/1229720013 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f045801c460 con 0x7f046412bad0 2024-09-21T09:43:27.356 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.336+0000 7fcc71da3640 1 --2- 172.21.15.92:0/3770039411 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcc6c095f70 0x7fcc6c122190 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fcc68009850 tx=0x7fcc680038a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:27.356 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.336+0000 7f0452ffd640 1 -- 172.21.15.92:0/1229720013 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f0458068cc0 con 0x7f046412bad0 2024-09-21T09:43:27.356 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.336+0000 7fcc627fc640 1 -- 172.21.15.92:0/3770039411 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcc6803f070 con 0x7fcc6c095f70 2024-09-21T09:43:27.357 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.336+0000 7f0452ffd640 1 -- 172.21.15.92:0/1229720013 --> v1:172.21.15.92:6805/7087154 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f0458015e50 con 0x7f0444057d30 2024-09-21T09:43:27.357 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.336+0000 7fcc74007640 1 -- 172.21.15.92:0/3770039411 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fcc6c074950 con 0x7fcc6c095f70 2024-09-21T09:43:27.357 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.336+0000 7fcc627fc640 1 -- 172.21.15.92:0/3770039411 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fcc680047a0 con 0x7fcc6c095f70 2024-09-21T09:43:27.357 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.336+0000 7f0452ffd640 1 -- 172.21.15.92:0/1229720013 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7f045806d030 con 0x7f046412bad0 2024-09-21T09:43:27.358 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.336+0000 7f7af12a5640 1 -- 172.21.15.92:0/71156227 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f7aec12c280 con 0x7f7aec0700b0 2024-09-21T09:43:27.358 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.336+0000 7fcc627fc640 1 -- 172.21.15.92:0/3770039411 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcc68035030 con 0x7fcc6c095f70 2024-09-21T09:43:27.358 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.337+0000 7fcc74007640 1 -- 172.21.15.92:0/3770039411 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fcc6c125720 con 0x7fcc6c095f70 2024-09-21T09:43:27.358 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.337+0000 7f7ad37fe640 1 -- 172.21.15.92:0/71156227 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f7adc005450 con 0x7f7aec0700b0 2024-09-21T09:43:27.359 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.336+0000 7f44a5d9b640 1 --2- 172.21.15.92:0/2141194680 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f44a812eec0 0x7f44a812a1f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.359 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.337+0000 7f44a5d9b640 1 -- 172.21.15.92:0/2141194680 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f44a8098bc0 msgr2=0x7f44a8129770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.359 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.337+0000 7f44a5d9b640 1 --2- 172.21.15.92:0/2141194680 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f44a8098bc0 0x7f44a8129770 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.359 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.337+0000 7f44a5d9b640 1 -- 172.21.15.92:0/2141194680 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f44a812b7a0 con 0x7f44a80994e0 2024-09-21T09:43:27.359 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.337+0000 7f44a5d9b640 1 --2- 172.21.15.92:0/2141194680 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f44a80994e0 0x7f44a8129cb0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f449c00e9f0 tx=0x7f449c00eeb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:27.359 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.337+0000 7f4496ffd640 1 -- 172.21.15.92:0/2141194680 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f449c00ccc0 con 0x7f44a80994e0 2024-09-21T09:43:27.359 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.337+0000 7f4496ffd640 1 -- 172.21.15.92:0/2141194680 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f449c004020 con 0x7f44a80994e0 2024-09-21T09:43:27.359 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.337+0000 7f4496ffd640 1 -- 172.21.15.92:0/2141194680 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f449c004470 con 0x7f44a80994e0 2024-09-21T09:43:27.359 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.337+0000 7f7ad37fe640 1 -- 172.21.15.92:0/71156227 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f7adc0699f0 con 0x7f7aec0700b0 2024-09-21T09:43:27.360 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.338+0000 7f44a6d9d640 1 --2- 172.21.15.92:0/2141194680 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f44a812eec0 0x7f44a812a1f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:27.360 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.338+0000 7f44acfa5640 1 -- 172.21.15.92:0/2141194680 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f44a81302d0 con 0x7f44a80994e0 2024-09-21T09:43:27.360 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.338+0000 7f44a659c640 1 --2- 172.21.15.92:0/2141194680 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f44a8098bc0 0x7f44a8129770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:27.360 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.338+0000 7f7ad37fe640 1 -- 172.21.15.92:0/71156227 --> v1:172.21.15.92:6813/138232475 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f7adc00e1a0 con 0x7f7ad8057da0 2024-09-21T09:43:27.361 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.338+0000 7fcc4ffff640 1 -- 172.21.15.92:0/3770039411 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_get_version(what=osdmap handle=1) -- 0x7fcc48001030 con 0x7fcc6c095f70 2024-09-21T09:43:27.361 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.338+0000 7f7ad37fe640 1 -- 172.21.15.92:0/71156227 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7f7adc070050 con 0x7f7aec0700b0 2024-09-21T09:43:27.361 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.338+0000 7f0452ffd640 1 -- 172.21.15.92:0/1229720013 <== osd.0 v1:172.21.15.92:6805/7087154 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f0458015e50 con 0x7f0444057d30 2024-09-21T09:43:27.361 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.338+0000 7f44acfa5640 1 -- 172.21.15.92:0/2141194680 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f44a812a730 con 0x7f44a80994e0 2024-09-21T09:43:27.362 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.339+0000 7fcc627fc640 1 -- 172.21.15.92:0/3770039411 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7fcc68042070 con 0x7fcc6c095f70 2024-09-21T09:43:27.362 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.339+0000 7f4496ffd640 1 -- 172.21.15.92:0/2141194680 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f449c005020 con 0x7f44a80994e0 2024-09-21T09:43:27.362 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.339+0000 7f44acfa5640 1 -- 172.21.15.92:0/2141194680 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f44a8098fa0 con 0x7f44a80994e0 2024-09-21T09:43:27.362 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.339+0000 7fcc627fc640 1 -- 172.21.15.92:0/3770039411 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fcc6808a480 con 0x7fcc6c095f70 2024-09-21T09:43:27.363 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.339+0000 7f7ad37fe640 1 -- 172.21.15.92:0/71156227 <== osd.3 v1:172.21.15.92:6813/138232475 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f7adc00e1a0 con 0x7f7ad8057da0 2024-09-21T09:43:27.363 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.339+0000 7fcc627fc640 1 -- 172.21.15.92:0/3770039411 --> v1:172.21.15.92:6809/4235634095 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7fcc5005ba50 con 0x7fcc50057da0 2024-09-21T09:43:27.363 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.339+0000 7fcc627fc640 1 -- 172.21.15.92:0/3770039411 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7fcc6808a870 con 0x7fcc6c095f70 2024-09-21T09:43:27.363 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.340+0000 7f4496ffd640 1 -- 172.21.15.92:0/2141194680 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f449c069770 con 0x7f44a80994e0 2024-09-21T09:43:27.363 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.340+0000 7f4496ffd640 1 -- 172.21.15.92:0/2141194680 --> v1:172.21.15.92:6801/2577383826 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f449c001d30 con 0x7f4488057d30 2024-09-21T09:43:27.364 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.340+0000 7f4496ffd640 1 -- 172.21.15.92:0/2141194680 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7f449c042670 con 0x7f44a80994e0 2024-09-21T09:43:27.364 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.340+0000 7fcc627fc640 1 -- 172.21.15.92:0/3770039411 <== osd.2 v1:172.21.15.92:6809/4235634095 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7fcc5005ba50 con 0x7fcc50057da0 2024-09-21T09:43:27.364 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.341+0000 7f4496ffd640 1 -- 172.21.15.92:0/2141194680 <== osd.1 v1:172.21.15.92:6801/2577383826 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f449c001d30 con 0x7f4488057d30 2024-09-21T09:43:27.364 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.354+0000 7f0468a70640 1 -- 172.21.15.92:0/1229720013 --> v1:172.21.15.92:6805/7087154 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f046406fb00 con 0x7f0444057d30 2024-09-21T09:43:27.365 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.354+0000 7f0452ffd640 1 -- 172.21.15.92:0/1229720013 <== osd.0 v1:172.21.15.92:6805/7087154 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (unknown 0 0 3581622916) 0x7f046406fb00 con 0x7f0444057d30 2024-09-21T09:43:27.365 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.354+0000 7f0468a70640 1 -- 172.21.15.92:0/1229720013 >> v1:172.21.15.92:6805/7087154 conn(0x7f0444057d30 legacy=0x7f044405a170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.365 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.355+0000 7f0468a70640 1 -- 172.21.15.92:0/1229720013 >> v1:172.21.15.92:6800/2299163855 conn(0x7f04440514b0 legacy=0x7f0444053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.365 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.355+0000 7f0468a70640 1 -- 172.21.15.92:0/1229720013 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f046412bad0 msgr2=0x7f046406edd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.365 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.355+0000 7f0468a70640 1 --2- 172.21.15.92:0/1229720013 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f046412bad0 0x7f046406edd0 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f045800c400 tx=0x7f045800c8c0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.366 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.355+0000 7f0462d9d640 1 -- 172.21.15.92:0/1229720013 reap_dead start 2024-09-21T09:43:27.366 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.355+0000 7f7af12a5640 1 -- 172.21.15.92:0/71156227 --> v1:172.21.15.92:6813/138232475 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f7aec1242e0 con 0x7f7ad8057da0 2024-09-21T09:43:27.366 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.355+0000 7f0468a70640 1 -- 172.21.15.92:0/1229720013 shutdown_connections 2024-09-21T09:43:27.366 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.355+0000 7f0468a70640 1 -- 172.21.15.92:0/1229720013 >> 172.21.15.92:0/1229720013 conn(0x7f046409afd0 msgr2=0x7f04641295a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:27.366 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.355+0000 7f0468a70640 1 -- 172.21.15.92:0/1229720013 shutdown_connections 2024-09-21T09:43:27.367 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.355+0000 7f0468a70640 1 -- 172.21.15.92:0/1229720013 wait complete. 2024-09-21T09:43:27.367 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.355+0000 7f7ad37fe640 1 -- 172.21.15.92:0/71156227 <== osd.3 v1:172.21.15.92:6813/138232475 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (unknown 0 0 2085588430) 0x7f7aec1242e0 con 0x7f7ad8057da0 2024-09-21T09:43:27.368 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.355+0000 7f7af12a5640 1 -- 172.21.15.92:0/71156227 >> v1:172.21.15.92:6813/138232475 conn(0x7f7ad8057da0 legacy=0x7f7ad805a1e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.368 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.355+0000 7f7af12a5640 1 -- 172.21.15.92:0/71156227 >> v1:172.21.15.92:6800/2299163855 conn(0x7f7ad80514b0 legacy=0x7f7ad8053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.368 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.355+0000 7f7af12a5640 1 -- 172.21.15.92:0/71156227 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7aec0700b0 msgr2=0x7f7aec123320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.368 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.355+0000 7f7af12a5640 1 --2- 172.21.15.92:0/71156227 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7aec0700b0 0x7f7aec123320 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f7adc00e9c0 tx=0x7f7adc00ee80 comp rx=0 tx=0).stop 2024-09-21T09:43:27.368 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.356+0000 7f7aeb7fe640 1 -- 172.21.15.92:0/71156227 reap_dead start 2024-09-21T09:43:27.368 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.356+0000 7f7af12a5640 1 -- 172.21.15.92:0/71156227 shutdown_connections 2024-09-21T09:43:27.368 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.356+0000 7f7af12a5640 1 -- 172.21.15.92:0/71156227 >> 172.21.15.92:0/71156227 conn(0x7f7aec09b9a0 msgr2=0x7f7aec099980 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:27.368 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.356+0000 7f7af12a5640 1 -- 172.21.15.92:0/71156227 shutdown_connections 2024-09-21T09:43:27.368 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.356+0000 7f7af12a5640 1 -- 172.21.15.92:0/71156227 wait complete. 2024-09-21T09:43:27.369 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.357+0000 7fcc4ffff640 1 -- 172.21.15.92:0/3770039411 --> v1:172.21.15.92:6809/4235634095 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7fcc48002dd0 con 0x7fcc50057da0 2024-09-21T09:43:27.369 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.357+0000 7fcc627fc640 1 -- 172.21.15.92:0/3770039411 <== osd.2 v1:172.21.15.92:6809/4235634095 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (unknown 0 0 1318614166) 0x7fcc48002dd0 con 0x7fcc50057da0 2024-09-21T09:43:27.369 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.357+0000 7f44acfa5640 1 -- 172.21.15.92:0/2141194680 --> v1:172.21.15.92:6801/2577383826 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f44a8063590 con 0x7f4488057d30 2024-09-21T09:43:27.370 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.357+0000 7fcc4ffff640 1 -- 172.21.15.92:0/3770039411 >> v1:172.21.15.92:6809/4235634095 conn(0x7fcc50057da0 legacy=0x7fcc5005a1e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.370 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.358+0000 7fcc4ffff640 1 -- 172.21.15.92:0/3770039411 >> v1:172.21.15.92:6800/2299163855 conn(0x7fcc500514b0 legacy=0x7fcc50053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.370 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.358+0000 7fcc4ffff640 1 -- 172.21.15.92:0/3770039411 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcc6c095f70 msgr2=0x7fcc6c122190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.370 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.358+0000 7fcc4ffff640 1 --2- 172.21.15.92:0/3770039411 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fcc6c095f70 0x7fcc6c122190 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fcc68009850 tx=0x7fcc680038a0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.370 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.358+0000 7fcc725a4640 1 -- 172.21.15.92:0/3770039411 reap_dead start 2024-09-21T09:43:27.370 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.358+0000 7fcc4ffff640 1 -- 172.21.15.92:0/3770039411 shutdown_connections 2024-09-21T09:43:27.370 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.358+0000 7fcc4ffff640 1 -- 172.21.15.92:0/3770039411 >> 172.21.15.92:0/3770039411 conn(0x7fcc6c09afd0 msgr2=0x7fcc6c099320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:27.370 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.358+0000 7f4496ffd640 1 -- 172.21.15.92:0/2141194680 <== osd.1 v1:172.21.15.92:6801/2577383826 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (unknown 0 0 3232526981) 0x7f44a8063590 con 0x7f4488057d30 2024-09-21T09:43:27.371 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.358+0000 7fcc4ffff640 1 -- 172.21.15.92:0/3770039411 shutdown_connections 2024-09-21T09:43:27.371 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.358+0000 7fcc4ffff640 1 -- 172.21.15.92:0/3770039411 wait complete. 2024-09-21T09:43:27.372 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.358+0000 7f44acfa5640 1 -- 172.21.15.92:0/2141194680 >> v1:172.21.15.92:6801/2577383826 conn(0x7f4488057d30 legacy=0x7f448805a170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.372 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.358+0000 7f44acfa5640 1 -- 172.21.15.92:0/2141194680 >> v1:172.21.15.92:6800/2299163855 conn(0x7f44880514b0 legacy=0x7f4488053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.372 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.358+0000 7f44acfa5640 1 -- 172.21.15.92:0/2141194680 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f44a80994e0 msgr2=0x7f44a8129cb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.372 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.358+0000 7f44acfa5640 1 --2- 172.21.15.92:0/2141194680 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f44a80994e0 0x7f44a8129cb0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f449c00e9f0 tx=0x7f449c00eeb0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.372 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.358+0000 7f44a6d9d640 1 -- 172.21.15.92:0/2141194680 reap_dead start 2024-09-21T09:43:27.372 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.358+0000 7f44acfa5640 1 -- 172.21.15.92:0/2141194680 shutdown_connections 2024-09-21T09:43:27.372 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.358+0000 7f44acfa5640 1 -- 172.21.15.92:0/2141194680 >> 172.21.15.92:0/2141194680 conn(0x7f44a80a1630 msgr2=0x7f44a80706e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:27.372 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.359+0000 7f44acfa5640 1 -- 172.21.15.92:0/2141194680 shutdown_connections 2024-09-21T09:43:27.373 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.359+0000 7f44acfa5640 1 -- 172.21.15.92:0/2141194680 wait complete. 2024-09-21T09:43:27.373 INFO:teuthology.orchestra.run.smithi092.stdout:115964116996 2024-09-21T09:43:27.373 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T09:43:27.374 INFO:teuthology.orchestra.run.smithi092.stdout:128849018883 2024-09-21T09:43:27.375 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T09:43:27.375 INFO:teuthology.orchestra.run.smithi092.stdout:124554051587 2024-09-21T09:43:27.376 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T09:43:27.376 INFO:teuthology.orchestra.run.smithi092.stdout:120259084292 2024-09-21T09:43:27.377 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T09:43:27.519 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.517+0000 7f456499e640 1 -- 172.21.15.92:0/3342801715 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4558030de0 con 0x7f45601323e0 2024-09-21T09:43:27.520 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.518+0000 7f45669a2640 1 -- 172.21.15.92:0/3342801715 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f45601323e0 msgr2=0x7f45601347d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.520 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.518+0000 7f45669a2640 1 --2- 172.21.15.92:0/3342801715 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f45601323e0 0x7f45601347d0 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f4558005fd0 tx=0x7f455802a980 comp rx=0 tx=0).stop 2024-09-21T09:43:27.520 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.518+0000 7f45669a2640 1 -- 172.21.15.92:0/3342801715 shutdown_connections 2024-09-21T09:43:27.521 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.518+0000 7f39fe0f2640 1 -- 172.21.15.92:0/3275577907 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f39f8098d40 msgr2=0x7f39f8099120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.521 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.518+0000 7f39fe0f2640 1 --2- 172.21.15.92:0/3275577907 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f39f8098d40 0x7f39f8099120 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f39e8009830 tx=0x7f39e802a3f0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.521 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.518+0000 7f39fe0f2640 1 -- 172.21.15.92:0/3275577907 shutdown_connections 2024-09-21T09:43:27.521 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.518+0000 7f39fe0f2640 1 --2- 172.21.15.92:0/3275577907 >> v2:172.21.15.92:3300/0 conn(0x7f39f812f1c0 0x7f39f812f600 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.521 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.519+0000 7f39fe0f2640 1 --2- 172.21.15.92:0/3275577907 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f39f8098d40 0x7f39f8099120 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.521 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.519+0000 7f39fe0f2640 1 -- 172.21.15.92:0/3275577907 >> 172.21.15.92:0/3275577907 conn(0x7f39f80a1790 msgr2=0x7f39f80a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:27.521 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.519+0000 7f39fe0f2640 1 -- 172.21.15.92:0/3275577907 shutdown_connections 2024-09-21T09:43:27.521 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.519+0000 7f39fe0f2640 1 -- 172.21.15.92:0/3275577907 wait complete. 2024-09-21T09:43:27.522 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.518+0000 7f45669a2640 1 --2- 172.21.15.92:0/3342801715 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f45601323e0 0x7f45601347d0 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.522 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.518+0000 7f45669a2640 1 --2- 172.21.15.92:0/3342801715 >> v2:172.21.15.92:3300/0 conn(0x7f456012fab0 0x7f4560131ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.522 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.518+0000 7f45669a2640 1 -- 172.21.15.92:0/3342801715 >> 172.21.15.92:0/3342801715 conn(0x7f456009b170 msgr2=0x7f456009b580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:27.522 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.518+0000 7f45669a2640 1 -- 172.21.15.92:0/3342801715 shutdown_connections 2024-09-21T09:43:27.522 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.519+0000 7fd19f7fe640 1 -- 172.21.15.92:0/1917429863 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd190033600 con 0x7fd1a0095d40 2024-09-21T09:43:27.523 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.519+0000 7f45669a2640 1 -- 172.21.15.92:0/3342801715 wait complete. 2024-09-21T09:43:27.523 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.519+0000 7fd1a6ccd640 1 -- 172.21.15.92:0/1917429863 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd1a0095d40 msgr2=0x7fd1a0096120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.524 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.519+0000 7fd1a6ccd640 1 --2- 172.21.15.92:0/1917429863 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd1a0095d40 0x7fd1a0096120 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7fd19000bc20 tx=0x7fd19002c460 comp rx=0 tx=0).stop 2024-09-21T09:43:27.524 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.520+0000 7fd1a6ccd640 1 -- 172.21.15.92:0/1917429863 shutdown_connections 2024-09-21T09:43:27.524 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.520+0000 7fd1a6ccd640 1 --2- 172.21.15.92:0/1917429863 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd1a012bc90 0x7fd1a012c120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.524 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.520+0000 7fd1a6ccd640 1 --2- 172.21.15.92:0/1917429863 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd1a0095d40 0x7fd1a0096120 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.524 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.520+0000 7f39fe0f2640 1 Processor -- start 2024-09-21T09:43:27.524 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.520+0000 7fd1a6ccd640 1 -- 172.21.15.92:0/1917429863 >> 172.21.15.92:0/1917429863 conn(0x7fd1a009c230 msgr2=0x7fd1a009c640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:27.524 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.520+0000 7fd1a6ccd640 1 -- 172.21.15.92:0/1917429863 shutdown_connections 2024-09-21T09:43:27.525 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.520+0000 7fd1a6ccd640 1 -- 172.21.15.92:0/1917429863 wait complete. 2024-09-21T09:43:27.525 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.520+0000 7f45669a2640 1 Processor -- start 2024-09-21T09:43:27.525 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.520+0000 7f45669a2640 1 -- start start 2024-09-21T09:43:27.526 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.520+0000 7f39fe0f2640 1 -- start start 2024-09-21T09:43:27.526 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.520+0000 7f39fe0f2640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f39f8099660 0x7f39f81c4f30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.526 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.520+0000 7f39fe0f2640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f39f812f1c0 0x7f39f81c5470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.526 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.520+0000 7f39fe0f2640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f39f80714f0 0x7f39f8071920 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.526 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.520+0000 7f39fe0f2640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f39f8137870 con 0x7f39f812f1c0 2024-09-21T09:43:27.526 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.520+0000 7f39fe0f2640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f39f81376f0 con 0x7f39f80714f0 2024-09-21T09:43:27.526 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.520+0000 7f39fe0f2640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f39f81379f0 con 0x7f39f8099660 2024-09-21T09:43:27.526 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f39f6ffd640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f39f812f1c0 0x7f39f81c5470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.526 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f39f77fe640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f39f8099660 0x7f39f81c4f30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.526 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f39f6ffd640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f39f812f1c0 0x7f39f81c5470 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:42836/0 (socket says 172.21.15.92:42836) 2024-09-21T09:43:27.526 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f39f6ffd640 1 -- 172.21.15.92:0/1656238817 learned_addr learned my addr 172.21.15.92:0/1656238817 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:27.527 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f39f7fff640 1 --2- 172.21.15.92:0/1656238817 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f39f80714f0 0x7f39f8071920 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.527 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7fd1a6ccd640 1 Processor -- start 2024-09-21T09:43:27.527 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7fd1a6ccd640 1 -- start start 2024-09-21T09:43:27.528 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f39f6ffd640 1 -- 172.21.15.92:0/1656238817 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f39f8099660 msgr2=0x7f39f81c4f30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.528 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f39f6ffd640 1 --2- 172.21.15.92:0/1656238817 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f39f8099660 0x7f39f81c4f30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.528 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f39f6ffd640 1 -- 172.21.15.92:0/1656238817 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f39f80714f0 msgr2=0x7f39f8071920 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.528 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f39f6ffd640 1 --2- 172.21.15.92:0/1656238817 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f39f80714f0 0x7f39f8071920 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.528 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f39f6ffd640 1 -- 172.21.15.92:0/1656238817 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f39f8136470 con 0x7f39f812f1c0 2024-09-21T09:43:27.528 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f39f7fff640 1 --2- 172.21.15.92:0/1656238817 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f39f80714f0 0x7f39f8071920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:27.528 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f45669a2640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4560129510 0x7f456012b4f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f39f6ffd640 1 --2- 172.21.15.92:0/1656238817 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f39f812f1c0 0x7f39f81c5470 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f39ec00c410 tx=0x7f39ec00c8d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:27.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f39f77fe640 1 --2- 172.21.15.92:0/1656238817 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f39f8099660 0x7f39f81c4f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T09:43:27.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f45659a0640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4560129510 0x7f456012b4f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.529 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f39dffff640 1 -- 172.21.15.92:0/1656238817 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f39ec017070 con 0x7f39f812f1c0 2024-09-21T09:43:27.530 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f39dffff640 1 -- 172.21.15.92:0/1656238817 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f39ec0035f0 con 0x7f39f812f1c0 2024-09-21T09:43:27.530 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f39fe0f2640 1 -- 172.21.15.92:0/1656238817 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f39f812f8a0 con 0x7f39f812f1c0 2024-09-21T09:43:27.530 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f39dffff640 1 -- 172.21.15.92:0/1656238817 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f39ec003a80 con 0x7f39f812f1c0 2024-09-21T09:43:27.530 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f4211ffb640 1 -- 172.21.15.92:0/2469243731 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4208038070 con 0x7f41fc004800 2024-09-21T09:43:27.530 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f39fe0f2640 1 -- 172.21.15.92:0/1656238817 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f39f8071e60 con 0x7f39f812f1c0 2024-09-21T09:43:27.531 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f45659a0640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4560129510 0x7f456012b4f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:44434/0 (socket says 172.21.15.92:44434) 2024-09-21T09:43:27.531 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7f45659a0640 1 -- 172.21.15.92:0/827799109 learned_addr learned my addr 172.21.15.92:0/827799109 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:27.531 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7f45669a2640 1 --2- 172.21.15.92:0/827799109 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f456012fab0 0x7f456012ba30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.531 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7f45669a2640 1 --2- 172.21.15.92:0/827799109 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f45601323e0 0x7f456012bf70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.531 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7f45669a2640 1 -- 172.21.15.92:0/827799109 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f4560135620 con 0x7f456012fab0 2024-09-21T09:43:27.531 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7f45669a2640 1 -- 172.21.15.92:0/827799109 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f45601354a0 con 0x7f4560129510 2024-09-21T09:43:27.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7fd1a6ccd640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd1a0095d40 0x7fd1a01c4cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7fd1a6ccd640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd1a0096660 0x7fd1a01c5210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7fd1a6ccd640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd1a012bc90 0x7fd1a0090390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7fd1a6ccd640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fd1a01343e0 con 0x7fd1a0096660 2024-09-21T09:43:27.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7fd1a6ccd640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fd1a0134260 con 0x7fd1a0095d40 2024-09-21T09:43:27.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.521+0000 7fd1a6ccd640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fd1a0134560 con 0x7fd1a012bc90 2024-09-21T09:43:27.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7fd19ffff640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd1a0096660 0x7fd1a01c5210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.532 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7fd1a526a640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd1a012bc90 0x7fd1a0090390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7fd19ffff640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd1a0096660 0x7fd1a01c5210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:42846/0 (socket says 172.21.15.92:42846) 2024-09-21T09:43:27.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7fd19ffff640 1 -- 172.21.15.92:0/4095858579 learned_addr learned my addr 172.21.15.92:0/4095858579 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:27.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7fd19ffff640 1 -- 172.21.15.92:0/4095858579 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd1a012bc90 msgr2=0x7fd1a0090390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7fd19ffff640 1 --2- 172.21.15.92:0/4095858579 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd1a012bc90 0x7fd1a0090390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7fd19ffff640 1 -- 172.21.15.92:0/4095858579 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd1a0095d40 msgr2=0x7fd1a01c4cd0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:43:27.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7fd19ffff640 1 --2- 172.21.15.92:0/4095858579 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd1a0095d40 0x7fd1a01c4cd0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7fd19ffff640 1 -- 172.21.15.92:0/4095858579 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd1a01242c0 con 0x7fd1a0096660 2024-09-21T09:43:27.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7fd1a526a640 1 --2- 172.21.15.92:0/4095858579 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd1a012bc90 0x7fd1a0090390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:27.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7f45669a2640 1 -- 172.21.15.92:0/827799109 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f45601357a0 con 0x7f45601323e0 2024-09-21T09:43:27.533 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7f456519f640 1 --2- 172.21.15.92:0/827799109 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f456012fab0 0x7f456012ba30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.534 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7f456519f640 1 -- 172.21.15.92:0/827799109 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f45601323e0 msgr2=0x7f456012bf70 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T09:43:27.534 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7f39fe0f2640 1 -- 172.21.15.92:0/1656238817 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f39f8099120 con 0x7f39f812f1c0 2024-09-21T09:43:27.534 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7f4219932640 1 -- 172.21.15.92:0/2469243731 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f41fc004800 msgr2=0x7f41fc004c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.534 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.523+0000 7fd19ffff640 1 --2- 172.21.15.92:0/4095858579 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd1a0096660 0x7fd1a01c5210 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7fd19400c490 tx=0x7fd19400c890 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:27.535 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7f456519f640 1 --2- 172.21.15.92:0/827799109 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f45601323e0 0x7f456012bf70 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.535 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7f4219932640 1 --2- 172.21.15.92:0/2469243731 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f41fc004800 0x7f41fc004c80 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f4208009b20 tx=0x7f41fc005830 comp rx=0 tx=0).stop 2024-09-21T09:43:27.535 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7f456519f640 1 -- 172.21.15.92:0/827799109 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4560129510 msgr2=0x7f456012b4f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.535 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7f4219932640 1 -- 172.21.15.92:0/2469243731 shutdown_connections 2024-09-21T09:43:27.536 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.523+0000 7f4219932640 1 --2- 172.21.15.92:0/2469243731 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4214096730 0x7f41fc0051c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.536 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.523+0000 7f4219932640 1 --2- 172.21.15.92:0/2469243731 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f41fc004800 0x7f41fc004c80 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.536 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.523+0000 7f4219932640 1 --2- 172.21.15.92:0/2469243731 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f41fc003a30 0x7f41fc003e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.536 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.523+0000 7f4219932640 1 -- 172.21.15.92:0/2469243731 >> 172.21.15.92:0/2469243731 conn(0x7f421409c470 msgr2=0x7f421409c880 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:27.536 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7f456519f640 1 --2- 172.21.15.92:0/827799109 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4560129510 0x7f456012b4f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.536 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.523+0000 7fd19d7fa640 1 -- 172.21.15.92:0/4095858579 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd194011790 con 0x7fd1a0096660 2024-09-21T09:43:27.536 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.522+0000 7f456519f640 1 -- 172.21.15.92:0/827799109 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f456012cc00 con 0x7f456012fab0 2024-09-21T09:43:27.537 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.523+0000 7f456519f640 1 --2- 172.21.15.92:0/827799109 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f456012fab0 0x7f456012ba30 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7f456006de00 tx=0x7f455000ee50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:27.537 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.523+0000 7fd19d7fa640 1 -- 172.21.15.92:0/4095858579 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fd194017070 con 0x7fd1a0096660 2024-09-21T09:43:27.537 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.523+0000 7fd19d7fa640 1 -- 172.21.15.92:0/4095858579 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd194011ce0 con 0x7fd1a0096660 2024-09-21T09:43:27.537 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.523+0000 7f39dffff640 1 -- 172.21.15.92:0/1656238817 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f39ec01a070 con 0x7f39f812f1c0 2024-09-21T09:43:27.537 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.523+0000 7f4219932640 1 -- 172.21.15.92:0/2469243731 shutdown_connections 2024-09-21T09:43:27.538 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.523+0000 7fd1a6ccd640 1 -- 172.21.15.92:0/4095858579 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd1a0098610 con 0x7fd1a0096660 2024-09-21T09:43:27.538 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.523+0000 7f4219932640 1 -- 172.21.15.92:0/2469243731 wait complete. 2024-09-21T09:43:27.538 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.523+0000 7fd1a6ccd640 1 -- 172.21.15.92:0/4095858579 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd1a01c5750 con 0x7fd1a0096660 2024-09-21T09:43:27.539 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.523+0000 7f45567fc640 1 -- 172.21.15.92:0/827799109 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f455000cc40 con 0x7f456012fab0 2024-09-21T09:43:27.539 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.523+0000 7f45567fc640 1 -- 172.21.15.92:0/827799109 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f455000cde0 con 0x7f456012fab0 2024-09-21T09:43:27.539 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.523+0000 7f45567fc640 1 -- 172.21.15.92:0/827799109 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4550004280 con 0x7f456012fab0 2024-09-21T09:43:27.539 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.523+0000 7f45669a2640 1 -- 172.21.15.92:0/827799109 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f456012c4b0 con 0x7f456012fab0 2024-09-21T09:43:27.539 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.523+0000 7f45669a2640 1 -- 172.21.15.92:0/827799109 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f456006ea80 con 0x7f456012fab0 2024-09-21T09:43:27.539 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.523+0000 7f39dffff640 1 -- 172.21.15.92:0/1656238817 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f39ec0682a0 con 0x7f39f812f1c0 2024-09-21T09:43:27.540 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.524+0000 7fd1a6ccd640 1 -- 172.21.15.92:0/4095858579 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd1a0096120 con 0x7fd1a0096660 2024-09-21T09:43:27.540 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.525+0000 7f4219932640 1 Processor -- start 2024-09-21T09:43:27.540 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.525+0000 7f45669a2640 1 -- 172.21.15.92:0/827799109 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f45601298f0 con 0x7f456012fab0 2024-09-21T09:43:27.542 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.525+0000 7f4219932640 1 -- start start 2024-09-21T09:43:27.542 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.525+0000 7f4219932640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f41fc003a30 0x7f421406f710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.542 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.525+0000 7f4219932640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f41fc004800 0x7f421406fc50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.542 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.525+0000 7f4219932640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4214096730 0x7f4214070190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:27.543 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.525+0000 7f4219932640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f421406e450 con 0x7f4214096730 2024-09-21T09:43:27.543 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.525+0000 7f4219932640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f421406e2d0 con 0x7f41fc003a30 2024-09-21T09:43:27.543 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.525+0000 7f4219932640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f421406e5d0 con 0x7f41fc004800 2024-09-21T09:43:27.543 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.526+0000 7f42137fe640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4214096730 0x7f4214070190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.543 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.526+0000 7f42127fc640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f41fc004800 0x7f421406fc50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.543 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.526+0000 7f42137fe640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4214096730 0x7f4214070190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:42862/0 (socket says 172.21.15.92:42862) 2024-09-21T09:43:27.543 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.526+0000 7f42137fe640 1 -- 172.21.15.92:0/921507386 learned_addr learned my addr 172.21.15.92:0/921507386 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:27.543 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.526+0000 7f4212ffd640 1 --2- 172.21.15.92:0/921507386 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f41fc003a30 0x7f421406f710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:27.543 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.526+0000 7f42127fc640 1 -- 172.21.15.92:0/921507386 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f41fc003a30 msgr2=0x7f421406f710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.543 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.526+0000 7f42127fc640 1 --2- 172.21.15.92:0/921507386 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f41fc003a30 0x7f421406f710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.543 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.526+0000 7f42127fc640 1 -- 172.21.15.92:0/921507386 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4214096730 msgr2=0x7f4214070190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.543 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.526+0000 7f42127fc640 1 --2- 172.21.15.92:0/921507386 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4214096730 0x7f4214070190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.543 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.526+0000 7f42127fc640 1 -- 172.21.15.92:0/921507386 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4214072810 con 0x7f41fc004800 2024-09-21T09:43:27.544 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.526+0000 7f42137fe640 1 --2- 172.21.15.92:0/921507386 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4214096730 0x7f4214070190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:27.544 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.526+0000 7f4212ffd640 1 --2- 172.21.15.92:0/921507386 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f41fc003a30 0x7f421406f710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:27.544 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.526+0000 7f42127fc640 1 --2- 172.21.15.92:0/921507386 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f41fc004800 0x7f421406fc50 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f4208032000 tx=0x7f4208033670 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:27.544 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.526+0000 7f41fbfff640 1 -- 172.21.15.92:0/921507386 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4208038040 con 0x7f41fc004800 2024-09-21T09:43:27.544 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.527+0000 7f41fbfff640 1 -- 172.21.15.92:0/921507386 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f4208005130 con 0x7f41fc004800 2024-09-21T09:43:27.544 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.527+0000 7f41fbfff640 1 -- 172.21.15.92:0/921507386 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4208009720 con 0x7f41fc004800 2024-09-21T09:43:27.544 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.527+0000 7f4219932640 1 -- 172.21.15.92:0/921507386 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4214126240 con 0x7f41fc004800 2024-09-21T09:43:27.544 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.527+0000 7f4219932640 1 -- 172.21.15.92:0/921507386 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f42140715c0 con 0x7f41fc004800 2024-09-21T09:43:27.544 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.528+0000 7f4219932640 1 -- 172.21.15.92:0/921507386 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f421412bc60 con 0x7f41fc004800 2024-09-21T09:43:27.544 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.528+0000 7f41fbfff640 1 -- 172.21.15.92:0/921507386 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f4208045020 con 0x7f41fc004800 2024-09-21T09:43:27.544 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.528+0000 7f39dffff640 1 -- 172.21.15.92:0/1656238817 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f39ec067350 con 0x7f39f812f1c0 2024-09-21T09:43:27.545 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.528+0000 7f41fbfff640 1 -- 172.21.15.92:0/921507386 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f42080896b0 con 0x7f41fc004800 2024-09-21T09:43:27.545 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.529+0000 7f45567fc640 1 -- 172.21.15.92:0/827799109 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f45500035f0 con 0x7f456012fab0 2024-09-21T09:43:27.545 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.529+0000 7fd19d7fa640 1 -- 172.21.15.92:0/4095858579 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7fd19401c460 con 0x7fd1a0096660 2024-09-21T09:43:27.545 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.530+0000 7fd19d7fa640 1 -- 172.21.15.92:0/4095858579 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fd194069860 con 0x7fd1a0096660 2024-09-21T09:43:27.545 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.530+0000 7f45567fc640 1 -- 172.21.15.92:0/827799109 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f4550069ba0 con 0x7f456012fab0 2024-09-21T09:43:27.546 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.533+0000 7f41fbfff640 1 -- 172.21.15.92:0/921507386 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f4208088760 con 0x7f41fc004800 2024-09-21T09:43:27.546 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.533+0000 7fd19d7fa640 1 -- 172.21.15.92:0/4095858579 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fd1940680b0 con 0x7fd1a0096660 2024-09-21T09:43:27.546 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.537+0000 7f45567fc640 1 -- 172.21.15.92:0/827799109 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f4550004420 con 0x7f456012fab0 2024-09-21T09:43:27.686 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.685+0000 7f39fe0f2640 1 -- 172.21.15.92:0/1656238817 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7f39f80698c0 con 0x7f39f812f1c0 2024-09-21T09:43:27.687 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.685+0000 7f39dffff640 1 -- 172.21.15.92:0/1656238817 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f39ec066e90 con 0x7f39f812f1c0 2024-09-21T09:43:27.687 INFO:teuthology.orchestra.run.smithi092.stdout:128849018882 2024-09-21T09:43:27.690 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.688+0000 7f39fe0f2640 1 -- 172.21.15.92:0/1656238817 >> v1:172.21.15.92:6800/2299163855 conn(0x7f39e40514b0 legacy=0x7f39e4053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.690 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.688+0000 7f39fe0f2640 1 -- 172.21.15.92:0/1656238817 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f39f812f1c0 msgr2=0x7f39f81c5470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.690 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.688+0000 7f39fe0f2640 1 --2- 172.21.15.92:0/1656238817 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f39f812f1c0 0x7f39f81c5470 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f39ec00c410 tx=0x7f39ec00c8d0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.690 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.688+0000 7f39fe0f2640 1 -- 172.21.15.92:0/1656238817 shutdown_connections 2024-09-21T09:43:27.690 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.688+0000 7f39fe0f2640 1 --2- 172.21.15.92:0/1656238817 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f39f80714f0 0x7f39f8071920 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.690 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.688+0000 7f39fe0f2640 1 --2- 172.21.15.92:0/1656238817 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f39f812f1c0 0x7f39f81c5470 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.691 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.688+0000 7f39fe0f2640 1 --2- 172.21.15.92:0/1656238817 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f39f8099660 0x7f39f81c4f30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.691 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.688+0000 7f39fe0f2640 1 -- 172.21.15.92:0/1656238817 >> 172.21.15.92:0/1656238817 conn(0x7f39f80a1790 msgr2=0x7f39f809d280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:27.691 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.688+0000 7f39fe0f2640 1 -- 172.21.15.92:0/1656238817 shutdown_connections 2024-09-21T09:43:27.691 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.688+0000 7f39fe0f2640 1 -- 172.21.15.92:0/1656238817 wait complete. 2024-09-21T09:43:27.691 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.689+0000 7fd1a6ccd640 1 -- 172.21.15.92:0/4095858579 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7fd1a0069a30 con 0x7fd1a0096660 2024-09-21T09:43:27.691 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.690+0000 7fd19d7fa640 1 -- 172.21.15.92:0/4095858579 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7fd194068430 con 0x7fd1a0096660 2024-09-21T09:43:27.692 INFO:teuthology.orchestra.run.smithi092.stdout:124554051586 2024-09-21T09:43:27.694 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.692+0000 7fd1a6ccd640 1 -- 172.21.15.92:0/4095858579 >> v1:172.21.15.92:6800/2299163855 conn(0x7fd18c0514e0 legacy=0x7fd18c0539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.694 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.692+0000 7fd1a6ccd640 1 -- 172.21.15.92:0/4095858579 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd1a0096660 msgr2=0x7fd1a01c5210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.694 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.692+0000 7fd1a6ccd640 1 --2- 172.21.15.92:0/4095858579 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd1a0096660 0x7fd1a01c5210 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7fd19400c490 tx=0x7fd19400c890 comp rx=0 tx=0).stop 2024-09-21T09:43:27.694 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.692+0000 7fd1a6ccd640 1 -- 172.21.15.92:0/4095858579 shutdown_connections 2024-09-21T09:43:27.694 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.692+0000 7fd1a6ccd640 1 --2- 172.21.15.92:0/4095858579 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fd1a012bc90 0x7fd1a0090390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.694 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.692+0000 7fd1a6ccd640 1 --2- 172.21.15.92:0/4095858579 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fd1a0096660 0x7fd1a01c5210 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.694 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.692+0000 7fd1a6ccd640 1 --2- 172.21.15.92:0/4095858579 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fd1a0095d40 0x7fd1a01c4cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.694 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.692+0000 7fd1a6ccd640 1 -- 172.21.15.92:0/4095858579 >> 172.21.15.92:0/4095858579 conn(0x7fd1a009c230 msgr2=0x7fd1a0098b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:27.694 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.692+0000 7fd1a6ccd640 1 -- 172.21.15.92:0/4095858579 shutdown_connections 2024-09-21T09:43:27.694 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.692+0000 7fd1a6ccd640 1 -- 172.21.15.92:0/4095858579 wait complete. 2024-09-21T09:43:27.695 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.693+0000 7f4219932640 1 -- 172.21.15.92:0/921507386 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7f42140697f0 con 0x7f41fc004800 2024-09-21T09:43:27.695 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.693+0000 7f41fbfff640 1 -- 172.21.15.92:0/921507386 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f420802e160 con 0x7f41fc004800 2024-09-21T09:43:27.695 INFO:teuthology.orchestra.run.smithi092.stdout:120259084290 2024-09-21T09:43:27.698 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.696+0000 7f4219932640 1 -- 172.21.15.92:0/921507386 >> v1:172.21.15.92:6800/2299163855 conn(0x7f42000514e0 legacy=0x7f42000539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.698 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.696+0000 7f4219932640 1 -- 172.21.15.92:0/921507386 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f41fc004800 msgr2=0x7f421406fc50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.698 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.696+0000 7f4219932640 1 --2- 172.21.15.92:0/921507386 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f41fc004800 0x7f421406fc50 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f4208032000 tx=0x7f4208033670 comp rx=0 tx=0).stop 2024-09-21T09:43:27.699 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.696+0000 7f4219932640 1 -- 172.21.15.92:0/921507386 shutdown_connections 2024-09-21T09:43:27.699 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.696+0000 7f4219932640 1 --2- 172.21.15.92:0/921507386 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4214096730 0x7f4214070190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.699 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.696+0000 7f4219932640 1 --2- 172.21.15.92:0/921507386 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f41fc004800 0x7f421406fc50 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.699 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.696+0000 7f4219932640 1 --2- 172.21.15.92:0/921507386 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f41fc003a30 0x7f421406f710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.699 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.696+0000 7f4219932640 1 -- 172.21.15.92:0/921507386 >> 172.21.15.92:0/921507386 conn(0x7f421409c470 msgr2=0x7f4214098b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:27.699 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.697+0000 7f4219932640 1 -- 172.21.15.92:0/921507386 shutdown_connections 2024-09-21T09:43:27.699 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.697+0000 7f4219932640 1 -- 172.21.15.92:0/921507386 wait complete. 2024-09-21T09:43:27.704 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.702+0000 7f45669a2640 1 -- 172.21.15.92:0/827799109 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7f45600697f0 con 0x7f456012fab0 2024-09-21T09:43:27.704 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.702+0000 7f45567fc640 1 -- 172.21.15.92:0/827799109 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f45500683f0 con 0x7f456012fab0 2024-09-21T09:43:27.704 INFO:teuthology.orchestra.run.smithi092.stdout:115964116994 2024-09-21T09:43:27.706 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.705+0000 7f45669a2640 1 -- 172.21.15.92:0/827799109 >> v1:172.21.15.92:6800/2299163855 conn(0x7f45440515b0 legacy=0x7f4544053a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.707 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.705+0000 7f45669a2640 1 -- 172.21.15.92:0/827799109 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f456012fab0 msgr2=0x7f456012ba30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:27.707 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.705+0000 7f45669a2640 1 --2- 172.21.15.92:0/827799109 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f456012fab0 0x7f456012ba30 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7f456006de00 tx=0x7f455000ee50 comp rx=0 tx=0).stop 2024-09-21T09:43:27.707 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.705+0000 7f45669a2640 1 -- 172.21.15.92:0/827799109 shutdown_connections 2024-09-21T09:43:27.707 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.705+0000 7f45669a2640 1 --2- 172.21.15.92:0/827799109 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f45601323e0 0x7f456012bf70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.707 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.705+0000 7f45669a2640 1 --2- 172.21.15.92:0/827799109 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f456012fab0 0x7f456012ba30 unknown :-1 s=CLOSED pgs=111 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.707 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.705+0000 7f45669a2640 1 --2- 172.21.15.92:0/827799109 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4560129510 0x7f456012b4f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:27.707 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.705+0000 7f45669a2640 1 -- 172.21.15.92:0/827799109 >> 172.21.15.92:0/827799109 conn(0x7f456009b170 msgr2=0x7f45601272b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:27.707 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.705+0000 7f45669a2640 1 -- 172.21.15.92:0/827799109 shutdown_connections 2024-09-21T09:43:27.707 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:27.705+0000 7f45669a2640 1 -- 172.21.15.92:0/827799109 wait complete. 2024-09-21T09:43:27.708 INFO:tasks.ceph.ceph_manager.ceph:need seq 128849018883 got 128849018882 for osd.3 2024-09-21T09:43:27.710 INFO:tasks.ceph.ceph_manager.ceph:need seq 124554051587 got 124554051586 for osd.2 2024-09-21T09:43:27.713 INFO:tasks.ceph.ceph_manager.ceph:need seq 120259084292 got 120259084290 for osd.1 2024-09-21T09:43:27.719 INFO:tasks.ceph.ceph_manager.ceph:need seq 115964116996 got 115964116994 for osd.0 2024-09-21T09:43:28.708 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T09:43:28.711 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T09:43:28.714 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T09:43:28.719 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T09:43:28.826 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.824+0000 7f2650dc3640 1 -- 172.21.15.92:0/3545047859 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f264c134310 msgr2=0x7f264c136700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:28.826 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.824+0000 7f2650dc3640 1 --2- 172.21.15.92:0/3545047859 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f264c134310 0x7f264c136700 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f2644005fd0 tx=0x7f264402a980 comp rx=0 tx=0).stop 2024-09-21T09:43:28.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.824+0000 7f2650dc3640 1 -- 172.21.15.92:0/3545047859 shutdown_connections 2024-09-21T09:43:28.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.824+0000 7f2650dc3640 1 --2- 172.21.15.92:0/3545047859 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f264c134310 0x7f264c136700 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:28.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.824+0000 7f2650dc3640 1 --2- 172.21.15.92:0/3545047859 >> v2:172.21.15.92:3300/0 conn(0x7f264c099830 0x7f264c133dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:28.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.824+0000 7f2650dc3640 1 -- 172.21.15.92:0/3545047859 >> 172.21.15.92:0/3545047859 conn(0x7f264c0a1790 msgr2=0x7f264c0a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:28.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.824+0000 7f2650dc3640 1 -- 172.21.15.92:0/3545047859 shutdown_connections 2024-09-21T09:43:28.827 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.824+0000 7f2650dc3640 1 -- 172.21.15.92:0/3545047859 wait complete. 2024-09-21T09:43:28.828 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.826+0000 7f2b621be640 1 -- 172.21.15.92:0/3649840406 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2b5c0973f0 msgr2=0x7f2b5c12be30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:28.828 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.826+0000 7f2b621be640 1 --2- 172.21.15.92:0/3649840406 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2b5c0973f0 0x7f2b5c12be30 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f2b54009c20 tx=0x7f2b5402c9f0 comp rx=0 tx=0).stop 2024-09-21T09:43:28.828 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.826+0000 7f2b621be640 1 -- 172.21.15.92:0/3649840406 shutdown_connections 2024-09-21T09:43:28.828 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.826+0000 7f2b621be640 1 --2- 172.21.15.92:0/3649840406 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2b5c0973f0 0x7f2b5c12be30 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:28.828 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.826+0000 7f2650dc3640 1 Processor -- start 2024-09-21T09:43:28.829 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.826+0000 7f2b621be640 1 --2- 172.21.15.92:0/3649840406 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2b5c095dd0 0x7f2b5c0961b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:28.829 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.826+0000 7f2b621be640 1 -- 172.21.15.92:0/3649840406 >> 172.21.15.92:0/3649840406 conn(0x7f2b5c09c470 msgr2=0x7f2b5c09c880 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:28.829 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.826+0000 7f2b621be640 1 -- 172.21.15.92:0/3649840406 shutdown_connections 2024-09-21T09:43:28.829 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.826+0000 7f2b621be640 1 -- 172.21.15.92:0/3649840406 wait complete. 2024-09-21T09:43:28.830 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.826+0000 7f2650dc3640 1 -- start start 2024-09-21T09:43:28.830 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.826+0000 7f2650dc3640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f264c098d40 0x7f264c1c4e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:28.830 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.826+0000 7f2650dc3640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f264c099830 0x7f264c1c53b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:28.830 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.827+0000 7f2650dc3640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f264c134310 0x7f264c12fd20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:28.830 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.827+0000 7f2650dc3640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f264c137890 con 0x7f264c098d40 2024-09-21T09:43:28.830 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.827+0000 7f2649d9b640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f264c099830 0x7f264c1c53b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:28.830 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.827+0000 7f2650dc3640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f264c137710 con 0x7f264c134310 2024-09-21T09:43:28.831 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.827+0000 7f2b621be640 1 Processor -- start 2024-09-21T09:43:28.831 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.827+0000 7f2650dc3640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f264c137a10 con 0x7f264c099830 2024-09-21T09:43:28.831 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.827+0000 7f2b621be640 1 -- start start 2024-09-21T09:43:28.832 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.827+0000 7f264a59c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f264c098d40 0x7f264c1c4e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:28.832 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.827+0000 7f264a59c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f264c098d40 0x7f264c1c4e70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:42886/0 (socket says 172.21.15.92:42886) 2024-09-21T09:43:28.832 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.827+0000 7f264a59c640 1 -- 172.21.15.92:0/3595073244 learned_addr learned my addr 172.21.15.92:0/3595073244 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:28.832 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.827+0000 7f264ad9d640 1 --2- 172.21.15.92:0/3595073244 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f264c134310 0x7f264c12fd20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:28.832 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.827+0000 7f264a59c640 1 -- 172.21.15.92:0/3595073244 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f264c099830 msgr2=0x7f264c1c53b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:28.832 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.827+0000 7f264a59c640 1 --2- 172.21.15.92:0/3595073244 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f264c099830 0x7f264c1c53b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:28.833 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.827+0000 7f264a59c640 1 -- 172.21.15.92:0/3595073244 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f264c134310 msgr2=0x7f264c12fd20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:28.833 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.827+0000 7f264a59c640 1 --2- 172.21.15.92:0/3595073244 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f264c134310 0x7f264c12fd20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:28.833 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.827+0000 7f264a59c640 1 -- 172.21.15.92:0/3595073244 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f264c133450 con 0x7f264c098d40 2024-09-21T09:43:28.833 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.827+0000 7f2649d9b640 1 --2- 172.21.15.92:0/3595073244 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f264c099830 0x7f264c1c53b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:28.833 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.827+0000 7f2b621be640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2b5c095dd0 0x7f2b5c093eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:28.833 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.828+0000 7f2b621be640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2b5c0966f0 0x7f2b5c0943f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:28.833 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.828+0000 7f2b621be640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2b5c0973f0 0x7f2b5c094930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:28.834 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.828+0000 7f2b621be640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f2b5c06eea0 con 0x7f2b5c0966f0 2024-09-21T09:43:28.834 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.828+0000 7f2b621be640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f2b5c06ed20 con 0x7f2b5c0973f0 2024-09-21T09:43:28.834 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.828+0000 7f2b621be640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f2b5c06f020 con 0x7f2b5c095dd0 2024-09-21T09:43:28.834 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.828+0000 7f2b5b7fe640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2b5c095dd0 0x7f2b5c093eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:28.834 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.828+0000 7f2b5b7fe640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2b5c095dd0 0x7f2b5c093eb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:48258/0 (socket says 172.21.15.92:48258) 2024-09-21T09:43:28.834 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.828+0000 7f2b5b7fe640 1 -- 172.21.15.92:0/4061403341 learned_addr learned my addr 172.21.15.92:0/4061403341 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:28.834 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.828+0000 7f264a59c640 1 --2- 172.21.15.92:0/3595073244 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f264c098d40 0x7f264c1c4e70 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f263c00c400 tx=0x7f263c00c8c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:28.835 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.828+0000 7f2b53fff640 1 --2- 172.21.15.92:0/4061403341 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2b5c0966f0 0x7f2b5c0943f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:28.835 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.828+0000 7f2b5bfff640 1 --2- 172.21.15.92:0/4061403341 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2b5c0973f0 0x7f2b5c094930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:28.835 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.828+0000 7f2b5b7fe640 1 -- 172.21.15.92:0/4061403341 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2b5c0973f0 msgr2=0x7f2b5c094930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:28.835 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.828+0000 7f263affd640 1 -- 172.21.15.92:0/3595073244 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f263c017070 con 0x7f264c098d40 2024-09-21T09:43:28.836 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.828+0000 7f263affd640 1 -- 172.21.15.92:0/3595073244 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f263c0053c0 con 0x7f264c098d40 2024-09-21T09:43:28.836 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.828+0000 7f263affd640 1 -- 172.21.15.92:0/3595073244 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f263c003850 con 0x7f264c098d40 2024-09-21T09:43:28.836 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.828+0000 7f2650dc3640 1 -- 172.21.15.92:0/3595073244 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f264c131500 con 0x7f264c098d40 2024-09-21T09:43:28.836 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.828+0000 7f2650dc3640 1 -- 172.21.15.92:0/3595073244 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f264c1c58f0 con 0x7f264c098d40 2024-09-21T09:43:28.836 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.828+0000 7f2b5b7fe640 1 --2- 172.21.15.92:0/4061403341 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2b5c0973f0 0x7f2b5c094930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:28.837 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.828+0000 7f2b5b7fe640 1 -- 172.21.15.92:0/4061403341 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2b5c0966f0 msgr2=0x7f2b5c0943f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:28.837 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.828+0000 7f2b5b7fe640 1 --2- 172.21.15.92:0/4061403341 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2b5c0966f0 0x7f2b5c0943f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:28.837 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.828+0000 7f2b5b7fe640 1 -- 172.21.15.92:0/4061403341 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2b5c08c260 con 0x7f2b5c095dd0 2024-09-21T09:43:28.837 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.829+0000 7f2b5bfff640 1 --2- 172.21.15.92:0/4061403341 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2b5c0973f0 0x7f2b5c094930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:43:28.837 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.829+0000 7f2b53fff640 1 --2- 172.21.15.92:0/4061403341 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2b5c0966f0 0x7f2b5c0943f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T09:43:28.837 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.829+0000 7f2b5b7fe640 1 --2- 172.21.15.92:0/4061403341 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2b5c095dd0 0x7f2b5c093eb0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f2b4c00e9f0 tx=0x7f2b4c00eeb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:28.837 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.829+0000 7f2b58ff9640 1 -- 172.21.15.92:0/4061403341 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2b4c00ccc0 con 0x7f2b5c095dd0 2024-09-21T09:43:28.837 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.829+0000 7f2b58ff9640 1 -- 172.21.15.92:0/4061403341 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2b4c00ce60 con 0x7f2b5c095dd0 2024-09-21T09:43:28.837 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.829+0000 7f2b621be640 1 -- 172.21.15.92:0/4061403341 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2b5c08def0 con 0x7f2b5c095dd0 2024-09-21T09:43:28.837 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.829+0000 7f2b58ff9640 1 -- 172.21.15.92:0/4061403341 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2b4c005280 con 0x7f2b5c095dd0 2024-09-21T09:43:28.837 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.829+0000 7f2b621be640 1 -- 172.21.15.92:0/4061403341 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f2b5c08ced0 con 0x7f2b5c095dd0 2024-09-21T09:43:28.838 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.830+0000 7f2650dc3640 1 -- 172.21.15.92:0/3595073244 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f264c099120 con 0x7f264c098d40 2024-09-21T09:43:28.838 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.830+0000 7f263affd640 1 -- 172.21.15.92:0/3595073244 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f263c01a070 con 0x7f264c098d40 2024-09-21T09:43:28.838 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.831+0000 7f2b621be640 1 -- 172.21.15.92:0/4061403341 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2b5c0961b0 con 0x7f2b5c095dd0 2024-09-21T09:43:28.838 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.831+0000 7f2b58ff9640 1 -- 172.21.15.92:0/4061403341 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f2b4c005420 con 0x7f2b5c095dd0 2024-09-21T09:43:28.839 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.831+0000 7f263affd640 1 -- 172.21.15.92:0/3595073244 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f263c068410 con 0x7f264c098d40 2024-09-21T09:43:28.839 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.831+0000 7f2b58ff9640 1 -- 172.21.15.92:0/4061403341 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f2b4c069a40 con 0x7f2b5c095dd0 2024-09-21T09:43:28.839 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.835+0000 7f2b58ff9640 1 -- 172.21.15.92:0/4061403341 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f2b4c0689d0 con 0x7f2b5c095dd0 2024-09-21T09:43:28.839 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.837+0000 7f263affd640 1 -- 172.21.15.92:0/3595073244 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f263c066c60 con 0x7f264c098d40 2024-09-21T09:43:28.870 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.868+0000 7fb942cb1640 1 -- 172.21.15.92:0/2828683476 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fb93c099660 msgr2=0x7fb93c12eab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:28.870 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.868+0000 7fb942cb1640 1 --2- 172.21.15.92:0/2828683476 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fb93c099660 0x7fb93c12eab0 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7fb928005fd0 tx=0x7fb92802a420 comp rx=0 tx=0).stop 2024-09-21T09:43:28.870 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.868+0000 7fb942cb1640 1 -- 172.21.15.92:0/2828683476 shutdown_connections 2024-09-21T09:43:28.870 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.868+0000 7fb942cb1640 1 --2- 172.21.15.92:0/2828683476 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fb93c12eff0 0x7fb93c12f480 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:28.870 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.868+0000 7fb942cb1640 1 --2- 172.21.15.92:0/2828683476 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fb93c099660 0x7fb93c12eab0 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:28.870 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.868+0000 7fb942cb1640 1 --2- 172.21.15.92:0/2828683476 >> v2:172.21.15.92:3300/0 conn(0x7fb93c098d40 0x7fb93c099120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:28.870 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.868+0000 7fb942cb1640 1 -- 172.21.15.92:0/2828683476 >> 172.21.15.92:0/2828683476 conn(0x7fb93c0a1790 msgr2=0x7fb93c0a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:28.870 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.868+0000 7fb942cb1640 1 -- 172.21.15.92:0/2828683476 shutdown_connections 2024-09-21T09:43:28.870 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.868+0000 7fb942cb1640 1 -- 172.21.15.92:0/2828683476 wait complete. 2024-09-21T09:43:28.873 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.869+0000 7fb942cb1640 1 Processor -- start 2024-09-21T09:43:28.873 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.869+0000 7fb942cb1640 1 -- start start 2024-09-21T09:43:28.873 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.869+0000 7fb942cb1640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fb93c098d40 0x7fb93c132b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:28.873 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.869+0000 7fb942cb1640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fb93c099660 0x7fb93c1330c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:28.873 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.869+0000 7fb940a4d640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fb93c098d40 0x7fb93c132b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:28.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.869+0000 7fb93bfff640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fb93c099660 0x7fb93c1330c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:28.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.869+0000 7fb940a4d640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fb93c098d40 0x7fb93c132b80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:48278/0 (socket says 172.21.15.92:48278) 2024-09-21T09:43:28.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.869+0000 7fb93bfff640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fb93c099660 0x7fb93c1330c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:44494/0 (socket says 172.21.15.92:44494) 2024-09-21T09:43:28.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.869+0000 7fb942cb1640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fb93c12eff0 0x7fb93c133600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:28.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.869+0000 7fb93bfff640 1 -- 172.21.15.92:0/2011476445 learned_addr learned my addr 172.21.15.92:0/2011476445 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:28.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.869+0000 7fb942cb1640 1 -- 172.21.15.92:0/2011476445 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fb93c136560 con 0x7fb93c12eff0 2024-09-21T09:43:28.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.869+0000 7fb942cb1640 1 -- 172.21.15.92:0/2011476445 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fb93c1363e0 con 0x7fb93c099660 2024-09-21T09:43:28.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.869+0000 7fb942cb1640 1 -- 172.21.15.92:0/2011476445 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fb93c1366e0 con 0x7fb93c098d40 2024-09-21T09:43:28.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.870+0000 7fb94124e640 1 --2- 172.21.15.92:0/2011476445 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fb93c12eff0 0x7fb93c133600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:28.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.870+0000 7fb93bfff640 1 -- 172.21.15.92:0/2011476445 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fb93c098d40 msgr2=0x7fb93c132b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:28.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.870+0000 7fb93bfff640 1 --2- 172.21.15.92:0/2011476445 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fb93c098d40 0x7fb93c132b80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:28.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.870+0000 7fb93bfff640 1 -- 172.21.15.92:0/2011476445 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fb93c12eff0 msgr2=0x7fb93c133600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:28.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.870+0000 7fb93bfff640 1 --2- 172.21.15.92:0/2011476445 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fb93c12eff0 0x7fb93c133600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:28.874 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.870+0000 7fb93bfff640 1 -- 172.21.15.92:0/2011476445 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb93c06e600 con 0x7fb93c099660 2024-09-21T09:43:28.875 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.870+0000 7fb94124e640 1 --2- 172.21.15.92:0/2011476445 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fb93c12eff0 0x7fb93c133600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:28.875 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.870+0000 7fb940a4d640 1 --2- 172.21.15.92:0/2011476445 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fb93c098d40 0x7fb93c132b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:28.875 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.870+0000 7fb93bfff640 1 --2- 172.21.15.92:0/2011476445 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fb93c099660 0x7fb93c1330c0 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7fb928000c00 tx=0x7fb9280038a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:28.875 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.871+0000 7fb9397fa640 1 -- 172.21.15.92:0/2011476445 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb92803f070 con 0x7fb93c099660 2024-09-21T09:43:28.875 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.871+0000 7fb9397fa640 1 -- 172.21.15.92:0/2011476445 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fb9280047a0 con 0x7fb93c099660 2024-09-21T09:43:28.875 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.871+0000 7fb9397fa640 1 -- 172.21.15.92:0/2011476445 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb928035030 con 0x7fb93c099660 2024-09-21T09:43:28.875 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.871+0000 7fb942cb1640 1 -- 172.21.15.92:0/2011476445 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb93c06e910 con 0x7fb93c099660 2024-09-21T09:43:28.875 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.871+0000 7fb942cb1640 1 -- 172.21.15.92:0/2011476445 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fb93c133b40 con 0x7fb93c099660 2024-09-21T09:43:28.876 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.872+0000 7fb942cb1640 1 -- 172.21.15.92:0/2011476445 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb93c12eab0 con 0x7fb93c099660 2024-09-21T09:43:28.876 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.872+0000 7fb9397fa640 1 -- 172.21.15.92:0/2011476445 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7fb928042070 con 0x7fb93c099660 2024-09-21T09:43:28.876 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.873+0000 7fb9397fa640 1 -- 172.21.15.92:0/2011476445 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fb92808a0e0 con 0x7fb93c099660 2024-09-21T09:43:28.879 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.877+0000 7fb9397fa640 1 -- 172.21.15.92:0/2011476445 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fb928089170 con 0x7fb93c099660 2024-09-21T09:43:28.880 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.878+0000 7f2cc4b49640 1 -- 172.21.15.92:0/1521635916 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2cc0099660 msgr2=0x7f2cc012eab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:28.880 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.878+0000 7f2cc4b49640 1 --2- 172.21.15.92:0/1521635916 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2cc0099660 0x7f2cc012eab0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f2cb4009900 tx=0x7f2cb402a4b0 comp rx=0 tx=0).stop 2024-09-21T09:43:28.880 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.878+0000 7f2cc4b49640 1 -- 172.21.15.92:0/1521635916 shutdown_connections 2024-09-21T09:43:28.880 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.878+0000 7f2cc4b49640 1 --2- 172.21.15.92:0/1521635916 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2cc0099660 0x7f2cc012eab0 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:28.881 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.878+0000 7f2cc4b49640 1 --2- 172.21.15.92:0/1521635916 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2cc0098d40 0x7f2cc0099120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:28.881 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.878+0000 7f2cc4b49640 1 -- 172.21.15.92:0/1521635916 >> 172.21.15.92:0/1521635916 conn(0x7f2cc00a1790 msgr2=0x7f2cc00a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:28.881 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.878+0000 7f2cc4b49640 1 -- 172.21.15.92:0/1521635916 shutdown_connections 2024-09-21T09:43:28.881 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.878+0000 7f2cc4b49640 1 -- 172.21.15.92:0/1521635916 wait complete. 2024-09-21T09:43:28.882 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.880+0000 7f2cc4b49640 1 Processor -- start 2024-09-21T09:43:28.882 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.880+0000 7f2cc4b49640 1 -- start start 2024-09-21T09:43:28.883 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.881+0000 7f2cc4b49640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2cc0098d40 0x7f2cc00710c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:28.883 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.881+0000 7f2cc4b49640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2cc0099660 0x7f2cc0071600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:28.884 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.881+0000 7f2cbe59c640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2cc0098d40 0x7f2cc00710c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:28.884 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.881+0000 7f2cbe59c640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2cc0098d40 0x7f2cc00710c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:48296/0 (socket says 172.21.15.92:48296) 2024-09-21T09:43:28.884 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.881+0000 7f2cbe59c640 1 -- 172.21.15.92:0/2135114161 learned_addr learned my addr 172.21.15.92:0/2135114161 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:28.884 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.881+0000 7f2cc4b49640 1 --2- 172.21.15.92:0/2135114161 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2cc012eff0 0x7f2cc0071b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:28.884 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.881+0000 7f2cc4b49640 1 -- 172.21.15.92:0/2135114161 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f2cc0137910 con 0x7f2cc012eff0 2024-09-21T09:43:28.884 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.881+0000 7f2cc4b49640 1 -- 172.21.15.92:0/2135114161 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f2cc0137790 con 0x7f2cc0099660 2024-09-21T09:43:28.884 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.881+0000 7f2cbdd9b640 1 --2- 172.21.15.92:0/2135114161 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2cc0099660 0x7f2cc0071600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:28.884 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.881+0000 7f2cbed9d640 1 --2- 172.21.15.92:0/2135114161 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2cc012eff0 0x7f2cc0071b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:28.884 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.882+0000 7f2cc4b49640 1 -- 172.21.15.92:0/2135114161 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f2cc0137a90 con 0x7f2cc0098d40 2024-09-21T09:43:28.885 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.882+0000 7f2cbed9d640 1 -- 172.21.15.92:0/2135114161 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2cc0098d40 msgr2=0x7f2cc00710c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:28.886 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.882+0000 7f2cbed9d640 1 --2- 172.21.15.92:0/2135114161 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2cc0098d40 0x7f2cc00710c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:28.886 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.882+0000 7f2cbed9d640 1 -- 172.21.15.92:0/2135114161 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2cc0099660 msgr2=0x7f2cc0071600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:28.886 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.882+0000 7f2cbed9d640 1 --2- 172.21.15.92:0/2135114161 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2cc0099660 0x7f2cc0071600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:28.886 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.882+0000 7f2cbed9d640 1 -- 172.21.15.92:0/2135114161 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2cc0073330 con 0x7f2cc012eff0 2024-09-21T09:43:28.886 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.882+0000 7f2cbdd9b640 1 --2- 172.21.15.92:0/2135114161 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2cc0099660 0x7f2cc0071600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:28.886 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.882+0000 7f2cbed9d640 1 --2- 172.21.15.92:0/2135114161 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2cc012eff0 0x7f2cc0071b40 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f2cb800c6f0 tx=0x7f2cb800cbb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:28.886 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.883+0000 7f2caeffd640 1 -- 172.21.15.92:0/2135114161 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2cb8003970 con 0x7f2cc012eff0 2024-09-21T09:43:28.886 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.883+0000 7f2caeffd640 1 -- 172.21.15.92:0/2135114161 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2cb8003b10 con 0x7f2cc012eff0 2024-09-21T09:43:28.886 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.883+0000 7f2cc4b49640 1 -- 172.21.15.92:0/2135114161 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2cc0073e10 con 0x7f2cc012eff0 2024-09-21T09:43:28.886 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.883+0000 7f2caeffd640 1 -- 172.21.15.92:0/2135114161 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2cb800a850 con 0x7f2cc012eff0 2024-09-21T09:43:28.886 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.883+0000 7f2cc4b49640 1 -- 172.21.15.92:0/2135114161 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f2cc0072260 con 0x7f2cc012eff0 2024-09-21T09:43:28.887 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.884+0000 7f2caeffd640 1 -- 172.21.15.92:0/2135114161 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f2cb800a9f0 con 0x7f2cc012eff0 2024-09-21T09:43:28.887 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.884+0000 7f2cc4b49640 1 -- 172.21.15.92:0/2135114161 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2cc0099120 con 0x7f2cc012eff0 2024-09-21T09:43:28.887 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.885+0000 7f2caeffd640 1 -- 172.21.15.92:0/2135114161 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f2cb8068e10 con 0x7f2cc012eff0 2024-09-21T09:43:28.893 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.892+0000 7f2caeffd640 1 -- 172.21.15.92:0/2135114161 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f2cb8067da0 con 0x7f2cc012eff0 2024-09-21T09:43:29.000 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.998+0000 7f2b621be640 1 -- 172.21.15.92:0/4061403341 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7f2b5c069950 con 0x7f2b5c095dd0 2024-09-21T09:43:29.000 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:28.998+0000 7f2b58ff9640 1 -- 172.21.15.92:0/4061403341 <== mon.2 v2:172.21.15.92:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f2b4c068590 con 0x7f2b5c095dd0 2024-09-21T09:43:29.000 INFO:teuthology.orchestra.run.smithi092.stdout:124554051587 2024-09-21T09:43:29.002 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.000+0000 7f2b621be640 1 -- 172.21.15.92:0/4061403341 >> v1:172.21.15.92:6800/2299163855 conn(0x7f2b3c0514b0 legacy=0x7f2b3c053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.002 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.000+0000 7f2b621be640 1 -- 172.21.15.92:0/4061403341 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2b5c095dd0 msgr2=0x7f2b5c093eb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.002 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.000+0000 7f2b621be640 1 --2- 172.21.15.92:0/4061403341 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2b5c095dd0 0x7f2b5c093eb0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f2b4c00e9f0 tx=0x7f2b4c00eeb0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.002 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.000+0000 7f2b621be640 1 -- 172.21.15.92:0/4061403341 shutdown_connections 2024-09-21T09:43:29.002 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.000+0000 7f2b621be640 1 --2- 172.21.15.92:0/4061403341 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2b5c0973f0 0x7f2b5c094930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.002 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.000+0000 7f2b621be640 1 --2- 172.21.15.92:0/4061403341 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2b5c0966f0 0x7f2b5c0943f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.002 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.000+0000 7f2b621be640 1 --2- 172.21.15.92:0/4061403341 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2b5c095dd0 0x7f2b5c093eb0 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.003 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.000+0000 7f2b621be640 1 -- 172.21.15.92:0/4061403341 >> 172.21.15.92:0/4061403341 conn(0x7f2b5c09c470 msgr2=0x7f2b5c099c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:29.003 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.000+0000 7f2b621be640 1 -- 172.21.15.92:0/4061403341 shutdown_connections 2024-09-21T09:43:29.003 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.000+0000 7f2b621be640 1 -- 172.21.15.92:0/4061403341 wait complete. 2024-09-21T09:43:29.014 INFO:tasks.ceph.ceph_manager.ceph:need seq 124554051587 got 124554051587 for osd.2 2024-09-21T09:43:29.014 DEBUG:teuthology.parallel:result is None 2024-09-21T09:43:29.035 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.033+0000 7f2650dc3640 1 -- 172.21.15.92:0/3595073244 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7f264c131db0 con 0x7f264c098d40 2024-09-21T09:43:29.035 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.033+0000 7f263affd640 1 -- 172.21.15.92:0/3595073244 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f263c066fe0 con 0x7f264c098d40 2024-09-21T09:43:29.035 INFO:teuthology.orchestra.run.smithi092.stdout:128849018883 2024-09-21T09:43:29.037 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.035+0000 7f2650dc3640 1 -- 172.21.15.92:0/3595073244 >> v1:172.21.15.92:6800/2299163855 conn(0x7f262c051500 legacy=0x7f262c0539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.037 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.035+0000 7f2650dc3640 1 -- 172.21.15.92:0/3595073244 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f264c098d40 msgr2=0x7f264c1c4e70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.037 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.035+0000 7f2650dc3640 1 --2- 172.21.15.92:0/3595073244 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f264c098d40 0x7f264c1c4e70 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f263c00c400 tx=0x7f263c00c8c0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.037 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.035+0000 7f2650dc3640 1 -- 172.21.15.92:0/3595073244 shutdown_connections 2024-09-21T09:43:29.037 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.035+0000 7f2650dc3640 1 --2- 172.21.15.92:0/3595073244 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f264c134310 0x7f264c12fd20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.038 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.035+0000 7f2650dc3640 1 --2- 172.21.15.92:0/3595073244 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f264c099830 0x7f264c1c53b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.038 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.035+0000 7f2650dc3640 1 --2- 172.21.15.92:0/3595073244 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f264c098d40 0x7f264c1c4e70 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.038 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.035+0000 7f2650dc3640 1 -- 172.21.15.92:0/3595073244 >> 172.21.15.92:0/3595073244 conn(0x7f264c0a1790 msgr2=0x7f264c128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:29.038 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.035+0000 7f2650dc3640 1 -- 172.21.15.92:0/3595073244 shutdown_connections 2024-09-21T09:43:29.038 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.035+0000 7f2650dc3640 1 -- 172.21.15.92:0/3595073244 wait complete. 2024-09-21T09:43:29.039 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.038+0000 7fb942cb1640 1 -- 172.21.15.92:0/2011476445 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7fb93c0698c0 con 0x7fb93c099660 2024-09-21T09:43:29.040 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.038+0000 7fb9397fa640 1 -- 172.21.15.92:0/2011476445 <== mon.1 v2:172.21.15.92:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7fb928088cb0 con 0x7fb93c099660 2024-09-21T09:43:29.040 INFO:teuthology.orchestra.run.smithi092.stdout:120259084292 2024-09-21T09:43:29.042 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.040+0000 7fb942cb1640 1 -- 172.21.15.92:0/2011476445 >> v1:172.21.15.92:6800/2299163855 conn(0x7fb9200514e0 legacy=0x7fb9200539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.042 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.040+0000 7fb942cb1640 1 -- 172.21.15.92:0/2011476445 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fb93c099660 msgr2=0x7fb93c1330c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.042 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.040+0000 7fb942cb1640 1 --2- 172.21.15.92:0/2011476445 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fb93c099660 0x7fb93c1330c0 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7fb928000c00 tx=0x7fb9280038a0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.042 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.040+0000 7fb942cb1640 1 -- 172.21.15.92:0/2011476445 shutdown_connections 2024-09-21T09:43:29.042 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.040+0000 7fb942cb1640 1 --2- 172.21.15.92:0/2011476445 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fb93c12eff0 0x7fb93c133600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.042 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.040+0000 7fb942cb1640 1 --2- 172.21.15.92:0/2011476445 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fb93c099660 0x7fb93c1330c0 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.042 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.040+0000 7fb942cb1640 1 --2- 172.21.15.92:0/2011476445 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fb93c098d40 0x7fb93c132b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.042 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.040+0000 7fb942cb1640 1 -- 172.21.15.92:0/2011476445 >> 172.21.15.92:0/2011476445 conn(0x7fb93c0a1790 msgr2=0x7fb93c128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:29.042 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.040+0000 7fb942cb1640 1 -- 172.21.15.92:0/2011476445 shutdown_connections 2024-09-21T09:43:29.043 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.040+0000 7fb942cb1640 1 -- 172.21.15.92:0/2011476445 wait complete. 2024-09-21T09:43:29.049 INFO:tasks.ceph.ceph_manager.ceph:need seq 128849018883 got 128849018883 for osd.3 2024-09-21T09:43:29.049 DEBUG:teuthology.parallel:result is None 2024-09-21T09:43:29.053 INFO:tasks.ceph.ceph_manager.ceph:need seq 120259084292 got 120259084292 for osd.1 2024-09-21T09:43:29.053 DEBUG:teuthology.parallel:result is None 2024-09-21T09:43:29.071 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.070+0000 7f2cc4b49640 1 -- 172.21.15.92:0/2135114161 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7f2cc00698c0 con 0x7f2cc012eff0 2024-09-21T09:43:29.072 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.070+0000 7f2caeffd640 1 -- 172.21.15.92:0/2135114161 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f2cb800ace0 con 0x7f2cc012eff0 2024-09-21T09:43:29.072 INFO:teuthology.orchestra.run.smithi092.stdout:115964116996 2024-09-21T09:43:29.074 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.072+0000 7f2cc4b49640 1 -- 172.21.15.92:0/2135114161 >> v1:172.21.15.92:6800/2299163855 conn(0x7f2ca00514b0 legacy=0x7f2ca0053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.074 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.072+0000 7f2cc4b49640 1 -- 172.21.15.92:0/2135114161 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2cc012eff0 msgr2=0x7f2cc0071b40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.074 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.072+0000 7f2cc4b49640 1 --2- 172.21.15.92:0/2135114161 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2cc012eff0 0x7f2cc0071b40 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7f2cb800c6f0 tx=0x7f2cb800cbb0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.074 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.072+0000 7f2cc4b49640 1 -- 172.21.15.92:0/2135114161 shutdown_connections 2024-09-21T09:43:29.074 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.072+0000 7f2cc4b49640 1 --2- 172.21.15.92:0/2135114161 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f2cc012eff0 0x7f2cc0071b40 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.074 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.072+0000 7f2cc4b49640 1 --2- 172.21.15.92:0/2135114161 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f2cc0099660 0x7f2cc0071600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.074 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.072+0000 7f2cc4b49640 1 --2- 172.21.15.92:0/2135114161 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f2cc0098d40 0x7f2cc00710c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.074 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.072+0000 7f2cc4b49640 1 -- 172.21.15.92:0/2135114161 >> 172.21.15.92:0/2135114161 conn(0x7f2cc00a1790 msgr2=0x7f2cc0128440 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:29.074 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.072+0000 7f2cc4b49640 1 -- 172.21.15.92:0/2135114161 shutdown_connections 2024-09-21T09:43:29.075 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.072+0000 7f2cc4b49640 1 -- 172.21.15.92:0/2135114161 wait complete. 2024-09-21T09:43:29.085 INFO:tasks.ceph.ceph_manager.ceph:need seq 115964116996 got 115964116996 for osd.0 2024-09-21T09:43:29.085 DEBUG:teuthology.parallel:result is None 2024-09-21T09:43:29.085 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean 2024-09-21T09:43:29.085 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T09:43:29.225 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.223+0000 7f98d6458640 1 -- 172.21.15.92:0/835232373 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f98d0132510 msgr2=0x7f98d0134900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.225 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.223+0000 7f98d6458640 1 --2- 172.21.15.92:0/835232373 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f98d0132510 0x7f98d0134900 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f98c8005fd0 tx=0x7f98c802aa10 comp rx=0 tx=0).stop 2024-09-21T09:43:29.225 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.223+0000 7f98d6458640 1 -- 172.21.15.92:0/835232373 shutdown_connections 2024-09-21T09:43:29.225 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.223+0000 7f98d6458640 1 --2- 172.21.15.92:0/835232373 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f98d0132510 0x7f98d0134900 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.225 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.223+0000 7f98d6458640 1 --2- 172.21.15.92:0/835232373 >> v2:172.21.15.92:3300/0 conn(0x7f98d012fbe0 0x7f98d0131fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.226 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.223+0000 7f98d6458640 1 -- 172.21.15.92:0/835232373 >> 172.21.15.92:0/835232373 conn(0x7f98d009a2e0 msgr2=0x7f98d0097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:29.226 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.223+0000 7f98d6458640 1 -- 172.21.15.92:0/835232373 shutdown_connections 2024-09-21T09:43:29.226 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.223+0000 7f98d6458640 1 -- 172.21.15.92:0/835232373 wait complete. 2024-09-21T09:43:29.228 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.224+0000 7f98d6458640 1 Processor -- start 2024-09-21T09:43:29.228 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.224+0000 7f98d6458640 1 -- start start 2024-09-21T09:43:29.228 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.224+0000 7f98d6458640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f98d012cc30 0x7f98d01c2ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:29.228 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.224+0000 7f98d6458640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f98d012fbe0 0x7f98d01c3420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:29.228 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.224+0000 7f98d6458640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f98d0132510 0x7f98d0128100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:29.228 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.224+0000 7f98d6458640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f98d0135820 con 0x7f98d0132510 2024-09-21T09:43:29.228 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.224+0000 7f98d6458640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f98d01356a0 con 0x7f98d012fbe0 2024-09-21T09:43:29.228 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.224+0000 7f98d6458640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f98d01359a0 con 0x7f98d012cc30 2024-09-21T09:43:29.228 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.224+0000 7f98cffff640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f98d012cc30 0x7f98d01c2ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:29.228 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.224+0000 7f98cf7fe640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f98d012fbe0 0x7f98d01c3420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:29.229 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.224+0000 7f98d49f5640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f98d0132510 0x7f98d0128100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:29.229 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.224+0000 7f98cffff640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f98d012cc30 0x7f98d01c2ee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:48320/0 (socket says 172.21.15.92:48320) 2024-09-21T09:43:29.229 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.224+0000 7f98cffff640 1 -- 172.21.15.92:0/1663774218 learned_addr learned my addr 172.21.15.92:0/1663774218 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:29.229 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.225+0000 7f98cffff640 1 -- 172.21.15.92:0/1663774218 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f98d012fbe0 msgr2=0x7f98d01c3420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.229 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.225+0000 7f98cffff640 1 --2- 172.21.15.92:0/1663774218 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f98d012fbe0 0x7f98d01c3420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.229 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.225+0000 7f98cffff640 1 -- 172.21.15.92:0/1663774218 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f98d0132510 msgr2=0x7f98d0128100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.229 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.225+0000 7f98cffff640 1 --2- 172.21.15.92:0/1663774218 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f98d0132510 0x7f98d0128100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.229 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.225+0000 7f98cffff640 1 -- 172.21.15.92:0/1663774218 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f98d0122190 con 0x7f98d012cc30 2024-09-21T09:43:29.229 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.225+0000 7f98d49f5640 1 --2- 172.21.15.92:0/1663774218 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f98d0132510 0x7f98d0128100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:29.229 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.225+0000 7f98cffff640 1 --2- 172.21.15.92:0/1663774218 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f98d012cc30 0x7f98d01c2ee0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f98c000c500 tx=0x7f98c000c9c0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:29.229 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.225+0000 7f98ccff9640 1 -- 172.21.15.92:0/1663774218 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f98c00052d0 con 0x7f98d012cc30 2024-09-21T09:43:29.229 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.225+0000 7f98ccff9640 1 -- 172.21.15.92:0/1663774218 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f98c0005470 con 0x7f98d012cc30 2024-09-21T09:43:29.229 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.225+0000 7f98d6458640 1 -- 172.21.15.92:0/1663774218 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f98d01c3960 con 0x7f98d012cc30 2024-09-21T09:43:29.229 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.225+0000 7f98ccff9640 1 -- 172.21.15.92:0/1663774218 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f98c0012870 con 0x7f98d012cc30 2024-09-21T09:43:29.230 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.225+0000 7f98d6458640 1 -- 172.21.15.92:0/1663774218 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f98d0122a60 con 0x7f98d012cc30 2024-09-21T09:43:29.230 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.226+0000 7f98ccff9640 1 -- 172.21.15.92:0/1663774218 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f98c00035f0 con 0x7f98d012cc30 2024-09-21T09:43:29.230 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.226+0000 7f98d6458640 1 -- 172.21.15.92:0/1663774218 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f98d009c7f0 con 0x7f98d012cc30 2024-09-21T09:43:29.230 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.227+0000 7f98ccff9640 1 -- 172.21.15.92:0/1663774218 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f98c0068b40 con 0x7f98d012cc30 2024-09-21T09:43:29.232 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.230+0000 7f98ccff9640 1 -- 172.21.15.92:0/1663774218 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f98d009c7f0 con 0x7f98d012cc30 2024-09-21T09:43:29.380 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.379+0000 7f98d6458640 1 -- 172.21.15.92:0/1663774218 --> v1:172.21.15.92:6800/2299163855 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f98d012ffc0 con 0x7f98bc0514e0 2024-09-21T09:43:29.381 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.380+0000 7f98ccff9640 1 -- 172.21.15.92:0/1663774218 <== mgr.14102 v1:172.21.15.92:6800/2299163855 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32025 (unknown 2965378022 0 1158794090) 0x7f98d012ffc0 con 0x7f98bc0514e0 2024-09-21T09:43:29.382 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:29.382 INFO:teuthology.orchestra.run.smithi092.stderr:dumped all 2024-09-21T09:43:29.383 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.381+0000 7f98d6458640 1 -- 172.21.15.92:0/1663774218 >> v1:172.21.15.92:6800/2299163855 conn(0x7f98bc0514e0 legacy=0x7f98bc0539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.384 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.381+0000 7f98d6458640 1 -- 172.21.15.92:0/1663774218 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f98d012cc30 msgr2=0x7f98d01c2ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.384 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.381+0000 7f98d6458640 1 --2- 172.21.15.92:0/1663774218 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f98d012cc30 0x7f98d01c2ee0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f98c000c500 tx=0x7f98c000c9c0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.384 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.381+0000 7f98d6458640 1 -- 172.21.15.92:0/1663774218 shutdown_connections 2024-09-21T09:43:29.384 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.381+0000 7f98d6458640 1 --2- 172.21.15.92:0/1663774218 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f98d0132510 0x7f98d0128100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.384 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.381+0000 7f98d6458640 1 --2- 172.21.15.92:0/1663774218 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f98d012fbe0 0x7f98d01c3420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.384 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.381+0000 7f98d6458640 1 --2- 172.21.15.92:0/1663774218 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f98d012cc30 0x7f98d01c2ee0 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.384 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.381+0000 7f98d6458640 1 -- 172.21.15.92:0/1663774218 >> 172.21.15.92:0/1663774218 conn(0x7f98d009a2e0 msgr2=0x7f98d01317a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:29.384 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.382+0000 7f98d6458640 1 -- 172.21.15.92:0/1663774218 shutdown_connections 2024-09-21T09:43:29.384 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.382+0000 7f98d6458640 1 -- 172.21.15.92:0/1663774218 wait complete. 2024-09-21T09:43:29.394 INFO:teuthology.orchestra.run.smithi092.stdout:{"pg_ready":true,"pg_map":{"version":26,"stamp":"2024-09-21T09:43:28.275665+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112292,"kb_used_data":2240,"kb_used_omap":17,"kb_used_meta":109934,"kb_avail":377375068,"statfs":{"total":386547056640,"available":386432069632,"internally_reserved":0,"allocated":2293760,"data_stored":1558648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17726,"internal_metadata":112573122},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.966390"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":52,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350236+0000","last_change":"2024-09-21T09:43:21.330614+0000","last_active":"2024-09-21T09:43:26.350236+0000","last_peered":"2024-09-21T09:43:26.350236+0000","last_clean":"2024-09-21T09:43:26.350236+0000","last_became_active":"2024-09-21T09:43:21.330325+0000","last_became_peered":"2024-09-21T09:43:21.330325+0000","last_unstale":"2024-09-21T09:43:26.350236+0000","last_undegraded":"2024-09-21T09:43:26.350236+0000","last_fullsized":"2024-09-21T09:43:26.350236+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:17:00.614617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084231599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":42,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.720522+0000","last_change":"2024-09-21T09:43:19.309958+0000","last_active":"2024-09-21T09:43:26.720522+0000","last_peered":"2024-09-21T09:43:26.720522+0000","last_clean":"2024-09-21T09:43:26.720522+0000","last_became_active":"2024-09-21T09:43:19.309603+0000","last_became_peered":"2024-09-21T09:43:19.309603+0000","last_unstale":"2024-09-21T09:43:26.720522+0000","last_undegraded":"2024-09-21T09:43:26.720522+0000","last_fullsized":"2024-09-21T09:43:26.720522+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:26:08.037398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049204800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":52,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350258+0000","last_change":"2024-09-21T09:43:21.330875+0000","last_active":"2024-09-21T09:43:26.350258+0000","last_peered":"2024-09-21T09:43:26.350258+0000","last_clean":"2024-09-21T09:43:26.350258+0000","last_became_active":"2024-09-21T09:43:21.330619+0000","last_became_peered":"2024-09-21T09:43:21.330619+0000","last_unstale":"2024-09-21T09:43:26.350258+0000","last_undegraded":"2024-09-21T09:43:26.350258+0000","last_fullsized":"2024-09-21T09:43:26.350258+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:22:03.959366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080402800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":42,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.720283+0000","last_change":"2024-09-21T09:43:19.310001+0000","last_active":"2024-09-21T09:43:26.720283+0000","last_peered":"2024-09-21T09:43:26.720283+0000","last_clean":"2024-09-21T09:43:26.720283+0000","last_became_active":"2024-09-21T09:43:19.309659+0000","last_became_peered":"2024-09-21T09:43:19.309659+0000","last_unstale":"2024-09-21T09:43:26.720283+0000","last_undegraded":"2024-09-21T09:43:26.720283+0000","last_fullsized":"2024-09-21T09:43:26.720283+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:39.861259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000465849,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"18'2","reported_seq":50,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350048+0000","last_change":"2024-09-21T09:43:21.315323+0000","last_active":"2024-09-21T09:43:26.350048+0000","last_peered":"2024-09-21T09:43:26.350048+0000","last_clean":"2024-09-21T09:43:26.350048+0000","last_became_active":"2024-09-21T09:43:21.314821+0000","last_became_peered":"2024-09-21T09:43:21.314821+0000","last_unstale":"2024-09-21T09:43:26.350048+0000","last_undegraded":"2024-09-21T09:43:26.350048+0000","last_fullsized":"2024-09-21T09:43:26.350048+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:32.540545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010637209999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":39,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350506+0000","last_change":"2024-09-21T09:43:21.315433+0000","last_active":"2024-09-21T09:43:26.350506+0000","last_peered":"2024-09-21T09:43:26.350506+0000","last_clean":"2024-09-21T09:43:26.350506+0000","last_became_active":"2024-09-21T09:43:21.315100+0000","last_became_peered":"2024-09-21T09:43:21.315100+0000","last_unstale":"2024-09-21T09:43:26.350506+0000","last_undegraded":"2024-09-21T09:43:26.350506+0000","last_fullsized":"2024-09-21T09:43:26.350506+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:05:55.727692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049819100000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":48,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.349885+0000","last_change":"2024-09-21T09:43:21.315140+0000","last_active":"2024-09-21T09:43:26.349885+0000","last_peered":"2024-09-21T09:43:26.349885+0000","last_clean":"2024-09-21T09:43:26.349885+0000","last_became_active":"2024-09-21T09:43:21.314817+0000","last_became_peered":"2024-09-21T09:43:21.314817+0000","last_unstale":"2024-09-21T09:43:26.349885+0000","last_undegraded":"2024-09-21T09:43:26.349885+0000","last_fullsized":"2024-09-21T09:43:26.349885+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:25.622786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088008499999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"16'1","reported_seq":47,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350078+0000","last_change":"2024-09-21T09:43:21.315433+0000","last_active":"2024-09-21T09:43:26.350078+0000","last_peered":"2024-09-21T09:43:26.350078+0000","last_clean":"2024-09-21T09:43:26.350078+0000","last_became_active":"2024-09-21T09:43:21.315110+0000","last_became_peered":"2024-09-21T09:43:21.315110+0000","last_unstale":"2024-09-21T09:43:26.350078+0000","last_undegraded":"2024-09-21T09:43:26.350078+0000","last_fullsized":"2024-09-21T09:43:26.350078+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:42.627708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010430319999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"21'104","reported_seq":195,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.720214+0000","last_change":"2024-09-21T09:43:19.310088+0000","last_active":"2024-09-21T09:43:26.720214+0000","last_peered":"2024-09-21T09:43:26.720214+0000","last_clean":"2024-09-21T09:43:26.720214+0000","last_became_active":"2024-09-21T09:43:19.309749+0000","last_became_peered":"2024-09-21T09:43:19.309749+0000","last_unstale":"2024-09-21T09:43:26.720214+0000","last_undegraded":"2024-09-21T09:43:26.720214+0000","last_fullsized":"2024-09-21T09:43:26.720214+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:50.808452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:50.808452+0000","last_clean_scrub_stamp":"2024-09-21T09:41:50.808452+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:18:22.448558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":30,"seq":128849018883,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27820,"kb_used_data":276,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344020,"statfs":{"total":96636764160,"available":96608276480,"internally_reserved":0,"allocated":282624,"data_stored":96278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4436,"internal_metadata":28176044},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":29,"seq":124554051587,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27816,"kb_used_data":272,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344024,"statfs":{"total":96636764160,"available":96608280576,"internally_reserved":0,"allocated":278528,"data_stored":96259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4432,"internal_metadata":28176048},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":28,"seq":120259084292,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28336,"kb_used_data":856,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343504,"statfs":{"total":96636764160,"available":96607748096,"internally_reserved":0,"allocated":876544,"data_stored":686646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4431,"internal_metadata":28110513},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":27,"seq":115964116996,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28320,"kb_used_data":836,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343520,"statfs":{"total":96636764160,"available":96607764480,"internally_reserved":0,"allocated":856064,"data_stored":679465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4427,"internal_metadata":28110517},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:43:29.395 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T09:43:29.495 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.493+0000 7fbea7de3640 1 -- 172.21.15.92:0/3665109098 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbea012fbe0 msgr2=0x7fbea0131fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.495 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.493+0000 7fbea7de3640 1 --2- 172.21.15.92:0/3665109098 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbea012fbe0 0x7fbea0131fd0 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7fbe94009830 tx=0x7fbe9402a480 comp rx=0 tx=0).stop 2024-09-21T09:43:29.495 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.493+0000 7fbea7de3640 1 -- 172.21.15.92:0/3665109098 shutdown_connections 2024-09-21T09:43:29.495 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.493+0000 7fbea7de3640 1 --2- 172.21.15.92:0/3665109098 >> v2:172.21.15.92:3300/0 conn(0x7fbea0132510 0x7fbea0134900 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.496 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.493+0000 7fbea7de3640 1 --2- 172.21.15.92:0/3665109098 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbea012fbe0 0x7fbea0131fd0 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.496 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.493+0000 7fbea7de3640 1 -- 172.21.15.92:0/3665109098 >> 172.21.15.92:0/3665109098 conn(0x7fbea009a2e0 msgr2=0x7fbea0097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:29.496 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.493+0000 7fbea7de3640 1 -- 172.21.15.92:0/3665109098 shutdown_connections 2024-09-21T09:43:29.496 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.493+0000 7fbea7de3640 1 -- 172.21.15.92:0/3665109098 wait complete. 2024-09-21T09:43:29.498 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.494+0000 7fbea7de3640 1 Processor -- start 2024-09-21T09:43:29.498 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.494+0000 7fbea7de3640 1 -- start start 2024-09-21T09:43:29.498 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.494+0000 7fbea7de3640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbea012cc30 0x7fbea0128240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:29.498 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.494+0000 7fbea7de3640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbea012fbe0 0x7fbea0128780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:29.498 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.494+0000 7fbea7de3640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbea0132510 0x7fbea0128cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:29.498 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.494+0000 7fbea7de3640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fbea0135870 con 0x7fbea012cc30 2024-09-21T09:43:29.498 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.494+0000 7fbea7de3640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fbea01356f0 con 0x7fbea0132510 2024-09-21T09:43:29.498 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.494+0000 7fbea7de3640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fbea01359f0 con 0x7fbea012fbe0 2024-09-21T09:43:29.498 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.494+0000 7fbea5b7f640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbea012cc30 0x7fbea0128240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:29.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.494+0000 7fbea537e640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbea012fbe0 0x7fbea0128780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:29.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.494+0000 7fbea6380640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbea0132510 0x7fbea0128cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:29.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.494+0000 7fbea5b7f640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbea012cc30 0x7fbea0128240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:42968/0 (socket says 172.21.15.92:42968) 2024-09-21T09:43:29.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.494+0000 7fbea5b7f640 1 -- 172.21.15.92:0/683295778 learned_addr learned my addr 172.21.15.92:0/683295778 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:29.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.495+0000 7fbea6380640 1 -- 172.21.15.92:0/683295778 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbea012fbe0 msgr2=0x7fbea0128780 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.495+0000 7fbea6380640 1 --2- 172.21.15.92:0/683295778 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbea012fbe0 0x7fbea0128780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.495+0000 7fbea6380640 1 -- 172.21.15.92:0/683295778 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbea012cc30 msgr2=0x7fbea0128240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.495+0000 7fbea6380640 1 --2- 172.21.15.92:0/683295778 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbea012cc30 0x7fbea0128240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.495+0000 7fbea6380640 1 -- 172.21.15.92:0/683295778 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbea0129850 con 0x7fbea0132510 2024-09-21T09:43:29.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.495+0000 7fbea6380640 1 --2- 172.21.15.92:0/683295778 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbea0132510 0x7fbea0128cc0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7fbe98005fd0 tx=0x7fbe9800c800 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:29.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.495+0000 7fbe927fc640 1 -- 172.21.15.92:0/683295778 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbe98003970 con 0x7fbea0132510 2024-09-21T09:43:29.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.495+0000 7fbe927fc640 1 -- 172.21.15.92:0/683295778 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fbe98003c00 con 0x7fbea0132510 2024-09-21T09:43:29.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.495+0000 7fbea7de3640 1 -- 172.21.15.92:0/683295778 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbea0129200 con 0x7fbea0132510 2024-09-21T09:43:29.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.495+0000 7fbe927fc640 1 -- 172.21.15.92:0/683295778 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbe98004020 con 0x7fbea0132510 2024-09-21T09:43:29.500 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.495+0000 7fbea7de3640 1 -- 172.21.15.92:0/683295778 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fbea01228d0 con 0x7fbea0132510 2024-09-21T09:43:29.500 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.496+0000 7fbea7de3640 1 -- 172.21.15.92:0/683295778 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbea0098740 con 0x7fbea0132510 2024-09-21T09:43:29.500 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.496+0000 7fbe927fc640 1 -- 172.21.15.92:0/683295778 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7fbe9801d020 con 0x7fbea0132510 2024-09-21T09:43:29.500 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.496+0000 7fbe927fc640 1 -- 172.21.15.92:0/683295778 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fbe98069580 con 0x7fbea0132510 2024-09-21T09:43:29.501 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.500+0000 7fbe927fc640 1 -- 172.21.15.92:0/683295778 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fbea0098740 con 0x7fbea0132510 2024-09-21T09:43:29.648 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.646+0000 7fbea7de3640 1 -- 172.21.15.92:0/683295778 --> v1:172.21.15.92:6800/2299163855 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7fbea012d010 con 0x7fbe840514e0 2024-09-21T09:43:29.649 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.647+0000 7fbe927fc640 1 -- 172.21.15.92:0/683295778 <== mgr.14102 v1:172.21.15.92:6800/2299163855 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32025 (unknown 2965378022 0 1158794090) 0x7fbea012d010 con 0x7fbe840514e0 2024-09-21T09:43:29.649 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:29.650 INFO:teuthology.orchestra.run.smithi092.stderr:dumped all 2024-09-21T09:43:29.651 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.649+0000 7fbea7de3640 1 -- 172.21.15.92:0/683295778 >> v1:172.21.15.92:6800/2299163855 conn(0x7fbe840514e0 legacy=0x7fbe840539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.651 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.649+0000 7fbea7de3640 1 -- 172.21.15.92:0/683295778 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbea0132510 msgr2=0x7fbea0128cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.651 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.649+0000 7fbea7de3640 1 --2- 172.21.15.92:0/683295778 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbea0132510 0x7fbea0128cc0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7fbe98005fd0 tx=0x7fbe9800c800 comp rx=0 tx=0).stop 2024-09-21T09:43:29.651 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.649+0000 7fbea7de3640 1 -- 172.21.15.92:0/683295778 shutdown_connections 2024-09-21T09:43:29.651 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.649+0000 7fbea7de3640 1 --2- 172.21.15.92:0/683295778 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbea0132510 0x7fbea0128cc0 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.651 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.649+0000 7fbea7de3640 1 --2- 172.21.15.92:0/683295778 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbea012fbe0 0x7fbea0128780 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.652 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.649+0000 7fbea7de3640 1 --2- 172.21.15.92:0/683295778 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbea012cc30 0x7fbea0128240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.652 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.649+0000 7fbea7de3640 1 -- 172.21.15.92:0/683295778 >> 172.21.15.92:0/683295778 conn(0x7fbea009a2e0 msgr2=0x7fbea012a650 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:29.652 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.649+0000 7fbea7de3640 1 -- 172.21.15.92:0/683295778 shutdown_connections 2024-09-21T09:43:29.652 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.649+0000 7fbea7de3640 1 -- 172.21.15.92:0/683295778 wait complete. 2024-09-21T09:43:29.662 INFO:teuthology.orchestra.run.smithi092.stdout:{"pg_ready":true,"pg_map":{"version":26,"stamp":"2024-09-21T09:43:28.275665+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112292,"kb_used_data":2240,"kb_used_omap":17,"kb_used_meta":109934,"kb_avail":377375068,"statfs":{"total":386547056640,"available":386432069632,"internally_reserved":0,"allocated":2293760,"data_stored":1558648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17726,"internal_metadata":112573122},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.966390"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":52,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350236+0000","last_change":"2024-09-21T09:43:21.330614+0000","last_active":"2024-09-21T09:43:26.350236+0000","last_peered":"2024-09-21T09:43:26.350236+0000","last_clean":"2024-09-21T09:43:26.350236+0000","last_became_active":"2024-09-21T09:43:21.330325+0000","last_became_peered":"2024-09-21T09:43:21.330325+0000","last_unstale":"2024-09-21T09:43:26.350236+0000","last_undegraded":"2024-09-21T09:43:26.350236+0000","last_fullsized":"2024-09-21T09:43:26.350236+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:17:00.614617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084231599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":42,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.720522+0000","last_change":"2024-09-21T09:43:19.309958+0000","last_active":"2024-09-21T09:43:26.720522+0000","last_peered":"2024-09-21T09:43:26.720522+0000","last_clean":"2024-09-21T09:43:26.720522+0000","last_became_active":"2024-09-21T09:43:19.309603+0000","last_became_peered":"2024-09-21T09:43:19.309603+0000","last_unstale":"2024-09-21T09:43:26.720522+0000","last_undegraded":"2024-09-21T09:43:26.720522+0000","last_fullsized":"2024-09-21T09:43:26.720522+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:26:08.037398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049204800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":52,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350258+0000","last_change":"2024-09-21T09:43:21.330875+0000","last_active":"2024-09-21T09:43:26.350258+0000","last_peered":"2024-09-21T09:43:26.350258+0000","last_clean":"2024-09-21T09:43:26.350258+0000","last_became_active":"2024-09-21T09:43:21.330619+0000","last_became_peered":"2024-09-21T09:43:21.330619+0000","last_unstale":"2024-09-21T09:43:26.350258+0000","last_undegraded":"2024-09-21T09:43:26.350258+0000","last_fullsized":"2024-09-21T09:43:26.350258+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:22:03.959366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080402800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":42,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.720283+0000","last_change":"2024-09-21T09:43:19.310001+0000","last_active":"2024-09-21T09:43:26.720283+0000","last_peered":"2024-09-21T09:43:26.720283+0000","last_clean":"2024-09-21T09:43:26.720283+0000","last_became_active":"2024-09-21T09:43:19.309659+0000","last_became_peered":"2024-09-21T09:43:19.309659+0000","last_unstale":"2024-09-21T09:43:26.720283+0000","last_undegraded":"2024-09-21T09:43:26.720283+0000","last_fullsized":"2024-09-21T09:43:26.720283+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:39.861259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000465849,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"18'2","reported_seq":50,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350048+0000","last_change":"2024-09-21T09:43:21.315323+0000","last_active":"2024-09-21T09:43:26.350048+0000","last_peered":"2024-09-21T09:43:26.350048+0000","last_clean":"2024-09-21T09:43:26.350048+0000","last_became_active":"2024-09-21T09:43:21.314821+0000","last_became_peered":"2024-09-21T09:43:21.314821+0000","last_unstale":"2024-09-21T09:43:26.350048+0000","last_undegraded":"2024-09-21T09:43:26.350048+0000","last_fullsized":"2024-09-21T09:43:26.350048+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:32.540545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010637209999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":39,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350506+0000","last_change":"2024-09-21T09:43:21.315433+0000","last_active":"2024-09-21T09:43:26.350506+0000","last_peered":"2024-09-21T09:43:26.350506+0000","last_clean":"2024-09-21T09:43:26.350506+0000","last_became_active":"2024-09-21T09:43:21.315100+0000","last_became_peered":"2024-09-21T09:43:21.315100+0000","last_unstale":"2024-09-21T09:43:26.350506+0000","last_undegraded":"2024-09-21T09:43:26.350506+0000","last_fullsized":"2024-09-21T09:43:26.350506+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:05:55.727692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049819100000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":48,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.349885+0000","last_change":"2024-09-21T09:43:21.315140+0000","last_active":"2024-09-21T09:43:26.349885+0000","last_peered":"2024-09-21T09:43:26.349885+0000","last_clean":"2024-09-21T09:43:26.349885+0000","last_became_active":"2024-09-21T09:43:21.314817+0000","last_became_peered":"2024-09-21T09:43:21.314817+0000","last_unstale":"2024-09-21T09:43:26.349885+0000","last_undegraded":"2024-09-21T09:43:26.349885+0000","last_fullsized":"2024-09-21T09:43:26.349885+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:25.622786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088008499999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"16'1","reported_seq":47,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350078+0000","last_change":"2024-09-21T09:43:21.315433+0000","last_active":"2024-09-21T09:43:26.350078+0000","last_peered":"2024-09-21T09:43:26.350078+0000","last_clean":"2024-09-21T09:43:26.350078+0000","last_became_active":"2024-09-21T09:43:21.315110+0000","last_became_peered":"2024-09-21T09:43:21.315110+0000","last_unstale":"2024-09-21T09:43:26.350078+0000","last_undegraded":"2024-09-21T09:43:26.350078+0000","last_fullsized":"2024-09-21T09:43:26.350078+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:42.627708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010430319999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"21'104","reported_seq":195,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.720214+0000","last_change":"2024-09-21T09:43:19.310088+0000","last_active":"2024-09-21T09:43:26.720214+0000","last_peered":"2024-09-21T09:43:26.720214+0000","last_clean":"2024-09-21T09:43:26.720214+0000","last_became_active":"2024-09-21T09:43:19.309749+0000","last_became_peered":"2024-09-21T09:43:19.309749+0000","last_unstale":"2024-09-21T09:43:26.720214+0000","last_undegraded":"2024-09-21T09:43:26.720214+0000","last_fullsized":"2024-09-21T09:43:26.720214+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:50.808452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:50.808452+0000","last_clean_scrub_stamp":"2024-09-21T09:41:50.808452+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:18:22.448558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":30,"seq":128849018883,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27820,"kb_used_data":276,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344020,"statfs":{"total":96636764160,"available":96608276480,"internally_reserved":0,"allocated":282624,"data_stored":96278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4436,"internal_metadata":28176044},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":29,"seq":124554051587,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27816,"kb_used_data":272,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344024,"statfs":{"total":96636764160,"available":96608280576,"internally_reserved":0,"allocated":278528,"data_stored":96259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4432,"internal_metadata":28176048},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":28,"seq":120259084292,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28336,"kb_used_data":856,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343504,"statfs":{"total":96636764160,"available":96607748096,"internally_reserved":0,"allocated":876544,"data_stored":686646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4431,"internal_metadata":28110513},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":27,"seq":115964116996,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28320,"kb_used_data":836,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343520,"statfs":{"total":96636764160,"available":96607764480,"internally_reserved":0,"allocated":856064,"data_stored":679465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4427,"internal_metadata":28110517},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:43:29.663 INFO:tasks.ceph.ceph_manager.ceph:clean! 2024-09-21T09:43:29.663 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T09:43:29.663 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy 2024-09-21T09:43:29.664 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph health --format=json 2024-09-21T09:43:29.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.760+0000 7f7aaefa5640 1 -- 172.21.15.92:0/864097820 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7aa8132510 msgr2=0x7f7aa8134900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.760+0000 7f7aaefa5640 1 --2- 172.21.15.92:0/864097820 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7aa8132510 0x7f7aa8134900 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7f7aa0005fd0 tx=0x7f7aa002a820 comp rx=0 tx=0).stop 2024-09-21T09:43:29.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.760+0000 7f7aaefa5640 1 -- 172.21.15.92:0/864097820 shutdown_connections 2024-09-21T09:43:29.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.760+0000 7f7aaefa5640 1 --2- 172.21.15.92:0/864097820 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7aa8132510 0x7f7aa8134900 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.760+0000 7f7aaefa5640 1 --2- 172.21.15.92:0/864097820 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7aa812fbe0 0x7f7aa8131fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.760+0000 7f7aaefa5640 1 -- 172.21.15.92:0/864097820 >> 172.21.15.92:0/864097820 conn(0x7f7aa809a2e0 msgr2=0x7f7aa8097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:29.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.760+0000 7f7aaefa5640 1 -- 172.21.15.92:0/864097820 shutdown_connections 2024-09-21T09:43:29.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.760+0000 7f7aaefa5640 1 -- 172.21.15.92:0/864097820 wait complete. 2024-09-21T09:43:29.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.761+0000 7f7aaefa5640 1 Processor -- start 2024-09-21T09:43:29.763 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.761+0000 7f7aaefa5640 1 -- start start 2024-09-21T09:43:29.766 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.762+0000 7f7aaefa5640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7aa812cc30 0x7f7aa80910d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:29.766 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.762+0000 7f7aaefa5640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7aa812fbe0 0x7f7aa8091610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:29.766 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.762+0000 7f7aaefa5640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7aa8132510 0x7f7aa8091b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:29.766 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.762+0000 7f7aaefa5640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f7aa8135820 con 0x7f7aa8132510 2024-09-21T09:43:29.766 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.762+0000 7f7aaefa5640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f7aa81356a0 con 0x7f7aa812fbe0 2024-09-21T09:43:29.766 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.762+0000 7f7aaefa5640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f7aa81359a0 con 0x7f7aa812cc30 2024-09-21T09:43:29.766 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.762+0000 7f7aad542640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7aa8132510 0x7f7aa8091b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:29.766 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.762+0000 7f7aacd41640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7aa812cc30 0x7f7aa80910d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:29.767 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.762+0000 7f7aa7fff640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7aa812fbe0 0x7f7aa8091610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:29.767 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.762+0000 7f7aacd41640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7aa812cc30 0x7f7aa80910d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:48338/0 (socket says 172.21.15.92:48338) 2024-09-21T09:43:29.767 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.762+0000 7f7aacd41640 1 -- 172.21.15.92:0/1860271971 learned_addr learned my addr 172.21.15.92:0/1860271971 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:29.767 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.762+0000 7f7aad542640 1 -- 172.21.15.92:0/1860271971 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7aa812cc30 msgr2=0x7f7aa80910d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.767 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.762+0000 7f7aad542640 1 --2- 172.21.15.92:0/1860271971 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7aa812cc30 0x7f7aa80910d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.767 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.762+0000 7f7aad542640 1 -- 172.21.15.92:0/1860271971 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7aa812fbe0 msgr2=0x7f7aa8091610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.767 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.762+0000 7f7aad542640 1 --2- 172.21.15.92:0/1860271971 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7aa812fbe0 0x7f7aa8091610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.767 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.762+0000 7f7aad542640 1 -- 172.21.15.92:0/1860271971 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7aa8126810 con 0x7f7aa8132510 2024-09-21T09:43:29.767 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.762+0000 7f7aad542640 1 --2- 172.21.15.92:0/1860271971 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7aa8132510 0x7f7aa8091b50 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f7aa0009000 tx=0x7f7aa0032be0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:29.767 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.762+0000 7f7aa57fa640 1 -- 172.21.15.92:0/1860271971 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7aa0037040 con 0x7f7aa8132510 2024-09-21T09:43:29.767 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.762+0000 7f7aa57fa640 1 -- 172.21.15.92:0/1860271971 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f7aa0042070 con 0x7f7aa8132510 2024-09-21T09:43:29.767 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.762+0000 7f7aa57fa640 1 -- 172.21.15.92:0/1860271971 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7aa0004040 con 0x7f7aa8132510 2024-09-21T09:43:29.767 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.762+0000 7f7aaefa5640 1 -- 172.21.15.92:0/1860271971 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7aa8092990 con 0x7f7aa8132510 2024-09-21T09:43:29.767 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.762+0000 7f7aaefa5640 1 -- 172.21.15.92:0/1860271971 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f7aa81cac50 con 0x7f7aa8132510 2024-09-21T09:43:29.768 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.763+0000 7f7aaefa5640 1 -- 172.21.15.92:0/1860271971 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7aa812d010 con 0x7f7aa8132510 2024-09-21T09:43:29.768 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.764+0000 7f7aa57fa640 1 -- 172.21.15.92:0/1860271971 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f7aa0004200 con 0x7f7aa8132510 2024-09-21T09:43:29.768 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.764+0000 7f7aa57fa640 1 -- 172.21.15.92:0/1860271971 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f7aa0089b80 con 0x7f7aa8132510 2024-09-21T09:43:29.769 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.767+0000 7f7aa57fa640 1 -- 172.21.15.92:0/1860271971 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f7aa0088c30 con 0x7f7aa8132510 2024-09-21T09:43:29.941 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.939+0000 7f7aaefa5640 1 -- 172.21.15.92:0/1860271971 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "health", "format": "json"} v 0) -- 0x7f7aa80699c0 con 0x7f7aa8132510 2024-09-21T09:43:29.941 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.940+0000 7f7aa57fa640 1 -- 172.21.15.92:0/1860271971 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "health", "format": "json"}]=0 v0) ==== 72+0+46 (secure 0 0 0) 0x7f7aa0032450 con 0x7f7aa8132510 2024-09-21T09:43:29.941 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:29.942 INFO:teuthology.orchestra.run.smithi092.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T09:43:29.943 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.941+0000 7f7aaefa5640 1 -- 172.21.15.92:0/1860271971 >> v1:172.21.15.92:6800/2299163855 conn(0x7f7a8c0514d0 legacy=0x7f7a8c053990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.943 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.941+0000 7f7aaefa5640 1 -- 172.21.15.92:0/1860271971 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7aa8132510 msgr2=0x7f7aa8091b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:29.943 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.941+0000 7f7aaefa5640 1 --2- 172.21.15.92:0/1860271971 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7aa8132510 0x7f7aa8091b50 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f7aa0009000 tx=0x7f7aa0032be0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.943 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.941+0000 7f7aaefa5640 1 -- 172.21.15.92:0/1860271971 shutdown_connections 2024-09-21T09:43:29.944 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.941+0000 7f7aaefa5640 1 --2- 172.21.15.92:0/1860271971 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f7aa8132510 0x7f7aa8091b50 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.944 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.941+0000 7f7aaefa5640 1 --2- 172.21.15.92:0/1860271971 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f7aa812fbe0 0x7f7aa8091610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.944 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.941+0000 7f7aaefa5640 1 --2- 172.21.15.92:0/1860271971 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f7aa812cc30 0x7f7aa80910d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:29.944 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.941+0000 7f7aaefa5640 1 -- 172.21.15.92:0/1860271971 >> 172.21.15.92:0/1860271971 conn(0x7f7aa809a2e0 msgr2=0x7f7aa812d910 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:29.944 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.941+0000 7f7aaefa5640 1 -- 172.21.15.92:0/1860271971 shutdown_connections 2024-09-21T09:43:29.944 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:29.942+0000 7f7aaefa5640 1 -- 172.21.15.92:0/1860271971 wait complete. 2024-09-21T09:43:29.954 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy done 2024-09-21T09:43:29.954 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:43:29.965 INFO:teuthology.task.print:**** done ceph.restart 2024-09-21T09:43:29.965 DEBUG:teuthology.run_tasks:Unwinding manager ceph.restart 2024-09-21T09:43:30.002 DEBUG:teuthology.run_tasks:Unwinding manager ceph.restart 2024-09-21T09:43:30.036 DEBUG:teuthology.run_tasks:Unwinding manager install.upgrade 2024-09-21T09:43:30.070 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2024-09-21T09:43:30.102 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean 2024-09-21T09:43:30.102 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T09:43:30.204 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.202+0000 7fc2bf3a2640 1 -- 172.21.15.92:0/3097755359 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc2b8134400 msgr2=0x7fc2b81367f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:30.204 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.202+0000 7fc2bf3a2640 1 --2- 172.21.15.92:0/3097755359 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc2b8134400 0x7fc2b81367f0 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7fc2b0009be0 tx=0x7fc2b002a7c0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.204 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.202+0000 7fc2bf3a2640 1 -- 172.21.15.92:0/3097755359 shutdown_connections 2024-09-21T09:43:30.204 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.202+0000 7fc2bf3a2640 1 --2- 172.21.15.92:0/3097755359 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc2b8134400 0x7fc2b81367f0 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.204 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.202+0000 7fc2bf3a2640 1 --2- 172.21.15.92:0/3097755359 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc2b8131ad0 0x7fc2b8133ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.204 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.202+0000 7fc2bf3a2640 1 -- 172.21.15.92:0/3097755359 >> 172.21.15.92:0/3097755359 conn(0x7fc2b80a1850 msgr2=0x7fc2b809dd20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:30.204 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.202+0000 7fc2bf3a2640 1 -- 172.21.15.92:0/3097755359 shutdown_connections 2024-09-21T09:43:30.204 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.202+0000 7fc2bf3a2640 1 -- 172.21.15.92:0/3097755359 wait complete. 2024-09-21T09:43:30.207 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.202+0000 7fc2bf3a2640 1 Processor -- start 2024-09-21T09:43:30.207 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.203+0000 7fc2bf3a2640 1 -- start start 2024-09-21T09:43:30.207 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.203+0000 7fc2bf3a2640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc2b8097750 0x7fc2b8126cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:30.207 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.203+0000 7fc2bf3a2640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc2b8131ad0 0x7fc2b8127210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:30.207 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.203+0000 7fc2bf3a2640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc2b8134400 0x7fc2b8127750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:30.207 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.203+0000 7fc2bf3a2640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fc2b8137640 con 0x7fc2b8131ad0 2024-09-21T09:43:30.207 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.203+0000 7fc2bf3a2640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fc2b81374c0 con 0x7fc2b8134400 2024-09-21T09:43:30.207 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.203+0000 7fc2bf3a2640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fc2b81377c0 con 0x7fc2b8097750 2024-09-21T09:43:30.207 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.203+0000 7fc2bdb9f640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc2b8131ad0 0x7fc2b8127210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:30.207 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.203+0000 7fc2be3a0640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc2b8097750 0x7fc2b8126cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:30.208 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.203+0000 7fc2bdb9f640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc2b8131ad0 0x7fc2b8127210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:43008/0 (socket says 172.21.15.92:43008) 2024-09-21T09:43:30.208 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.203+0000 7fc2be3a0640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc2b8097750 0x7fc2b8126cd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:48360/0 (socket says 172.21.15.92:48360) 2024-09-21T09:43:30.208 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.203+0000 7fc2bdb9f640 1 -- 172.21.15.92:0/3403662574 learned_addr learned my addr 172.21.15.92:0/3403662574 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:30.208 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.203+0000 7fc2beba1640 1 --2- 172.21.15.92:0/3403662574 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc2b8134400 0x7fc2b8127750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:30.208 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.203+0000 7fc2be3a0640 1 -- 172.21.15.92:0/3403662574 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc2b8134400 msgr2=0x7fc2b8127750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:30.208 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.203+0000 7fc2be3a0640 1 --2- 172.21.15.92:0/3403662574 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc2b8134400 0x7fc2b8127750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.208 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.203+0000 7fc2be3a0640 1 -- 172.21.15.92:0/3403662574 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc2b8131ad0 msgr2=0x7fc2b8127210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:30.208 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.203+0000 7fc2be3a0640 1 --2- 172.21.15.92:0/3403662574 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc2b8131ad0 0x7fc2b8127210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.208 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.203+0000 7fc2be3a0640 1 -- 172.21.15.92:0/3403662574 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc2b8128620 con 0x7fc2b8097750 2024-09-21T09:43:30.208 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.203+0000 7fc2bdb9f640 1 --2- 172.21.15.92:0/3403662574 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc2b8131ad0 0x7fc2b8127210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:30.208 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.204+0000 7fc2be3a0640 1 --2- 172.21.15.92:0/3403662574 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc2b8097750 0x7fc2b8126cd0 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7fc2b400c3c0 tx=0x7fc2b400c880 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:30.208 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.204+0000 7fc2aeffd640 1 -- 172.21.15.92:0/3403662574 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc2b4011790 con 0x7fc2b8097750 2024-09-21T09:43:30.208 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.204+0000 7fc2aeffd640 1 -- 172.21.15.92:0/3403662574 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fc2b4017070 con 0x7fc2b8097750 2024-09-21T09:43:30.208 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.204+0000 7fc2bf3a2640 1 -- 172.21.15.92:0/3403662574 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc2b8128e70 con 0x7fc2b8097750 2024-09-21T09:43:30.209 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.204+0000 7fc2aeffd640 1 -- 172.21.15.92:0/3403662574 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc2b4011d20 con 0x7fc2b8097750 2024-09-21T09:43:30.209 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.204+0000 7fc2bf3a2640 1 -- 172.21.15.92:0/3403662574 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fc2b81299c0 con 0x7fc2b8097750 2024-09-21T09:43:30.209 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.205+0000 7fc2bf3a2640 1 -- 172.21.15.92:0/3403662574 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc2b8097b30 con 0x7fc2b8097750 2024-09-21T09:43:30.209 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.205+0000 7fc2aeffd640 1 -- 172.21.15.92:0/3403662574 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7fc2b401c460 con 0x7fc2b8097750 2024-09-21T09:43:30.209 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.205+0000 7fc2aeffd640 1 -- 172.21.15.92:0/3403662574 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fc2b4068c90 con 0x7fc2b8097750 2024-09-21T09:43:30.210 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.209+0000 7fc2aeffd640 1 -- 172.21.15.92:0/3403662574 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fc2b4067d40 con 0x7fc2b8097750 2024-09-21T09:43:30.358 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.357+0000 7fc2bf3a2640 1 -- 172.21.15.92:0/3403662574 --> v1:172.21.15.92:6800/2299163855 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7fc2b80697f0 con 0x7fc29c0514b0 2024-09-21T09:43:30.359 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.357+0000 7fc2aeffd640 1 -- 172.21.15.92:0/3403662574 <== mgr.14102 v1:172.21.15.92:6800/2299163855 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32001 (unknown 2965378022 0 2174376010) 0x7fc2b80697f0 con 0x7fc29c0514b0 2024-09-21T09:43:30.359 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:30.360 INFO:teuthology.orchestra.run.smithi092.stderr:dumped all 2024-09-21T09:43:30.361 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.359+0000 7fc2bf3a2640 1 -- 172.21.15.92:0/3403662574 >> v1:172.21.15.92:6800/2299163855 conn(0x7fc29c0514b0 legacy=0x7fc29c053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:30.361 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.359+0000 7fc2bf3a2640 1 -- 172.21.15.92:0/3403662574 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc2b8097750 msgr2=0x7fc2b8126cd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:30.361 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.359+0000 7fc2bf3a2640 1 --2- 172.21.15.92:0/3403662574 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc2b8097750 0x7fc2b8126cd0 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7fc2b400c3c0 tx=0x7fc2b400c880 comp rx=0 tx=0).stop 2024-09-21T09:43:30.361 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.359+0000 7fc2bf3a2640 1 -- 172.21.15.92:0/3403662574 shutdown_connections 2024-09-21T09:43:30.361 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.359+0000 7fc2bf3a2640 1 --2- 172.21.15.92:0/3403662574 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fc2b8134400 0x7fc2b8127750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.361 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.359+0000 7fc2bf3a2640 1 --2- 172.21.15.92:0/3403662574 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fc2b8131ad0 0x7fc2b8127210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.362 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.359+0000 7fc2bf3a2640 1 --2- 172.21.15.92:0/3403662574 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fc2b8097750 0x7fc2b8126cd0 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.362 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.359+0000 7fc2bf3a2640 1 -- 172.21.15.92:0/3403662574 >> 172.21.15.92:0/3403662574 conn(0x7fc2b80a1850 msgr2=0x7fc2b8131050 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:30.362 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.359+0000 7fc2bf3a2640 1 -- 172.21.15.92:0/3403662574 shutdown_connections 2024-09-21T09:43:30.362 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.359+0000 7fc2bf3a2640 1 -- 172.21.15.92:0/3403662574 wait complete. 2024-09-21T09:43:30.372 INFO:teuthology.orchestra.run.smithi092.stdout:{"pg_ready":true,"pg_map":{"version":27,"stamp":"2024-09-21T09:43:30.275953+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112292,"kb_used_data":2240,"kb_used_omap":17,"kb_used_meta":109934,"kb_avail":377375068,"statfs":{"total":386547056640,"available":386432069632,"internally_reserved":0,"allocated":2293760,"data_stored":1558648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17726,"internal_metadata":112573122},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001446"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":52,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350236+0000","last_change":"2024-09-21T09:43:21.330614+0000","last_active":"2024-09-21T09:43:26.350236+0000","last_peered":"2024-09-21T09:43:26.350236+0000","last_clean":"2024-09-21T09:43:26.350236+0000","last_became_active":"2024-09-21T09:43:21.330325+0000","last_became_peered":"2024-09-21T09:43:21.330325+0000","last_unstale":"2024-09-21T09:43:26.350236+0000","last_undegraded":"2024-09-21T09:43:26.350236+0000","last_fullsized":"2024-09-21T09:43:26.350236+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:17:00.614617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084231599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":42,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.720522+0000","last_change":"2024-09-21T09:43:19.309958+0000","last_active":"2024-09-21T09:43:26.720522+0000","last_peered":"2024-09-21T09:43:26.720522+0000","last_clean":"2024-09-21T09:43:26.720522+0000","last_became_active":"2024-09-21T09:43:19.309603+0000","last_became_peered":"2024-09-21T09:43:19.309603+0000","last_unstale":"2024-09-21T09:43:26.720522+0000","last_undegraded":"2024-09-21T09:43:26.720522+0000","last_fullsized":"2024-09-21T09:43:26.720522+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:26:08.037398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049204800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":52,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350258+0000","last_change":"2024-09-21T09:43:21.330875+0000","last_active":"2024-09-21T09:43:26.350258+0000","last_peered":"2024-09-21T09:43:26.350258+0000","last_clean":"2024-09-21T09:43:26.350258+0000","last_became_active":"2024-09-21T09:43:21.330619+0000","last_became_peered":"2024-09-21T09:43:21.330619+0000","last_unstale":"2024-09-21T09:43:26.350258+0000","last_undegraded":"2024-09-21T09:43:26.350258+0000","last_fullsized":"2024-09-21T09:43:26.350258+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:22:03.959366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080402800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":42,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.720283+0000","last_change":"2024-09-21T09:43:19.310001+0000","last_active":"2024-09-21T09:43:26.720283+0000","last_peered":"2024-09-21T09:43:26.720283+0000","last_clean":"2024-09-21T09:43:26.720283+0000","last_became_active":"2024-09-21T09:43:19.309659+0000","last_became_peered":"2024-09-21T09:43:19.309659+0000","last_unstale":"2024-09-21T09:43:26.720283+0000","last_undegraded":"2024-09-21T09:43:26.720283+0000","last_fullsized":"2024-09-21T09:43:26.720283+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:39.861259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000465849,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"18'2","reported_seq":50,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350048+0000","last_change":"2024-09-21T09:43:21.315323+0000","last_active":"2024-09-21T09:43:26.350048+0000","last_peered":"2024-09-21T09:43:26.350048+0000","last_clean":"2024-09-21T09:43:26.350048+0000","last_became_active":"2024-09-21T09:43:21.314821+0000","last_became_peered":"2024-09-21T09:43:21.314821+0000","last_unstale":"2024-09-21T09:43:26.350048+0000","last_undegraded":"2024-09-21T09:43:26.350048+0000","last_fullsized":"2024-09-21T09:43:26.350048+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:32.540545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010637209999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":39,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350506+0000","last_change":"2024-09-21T09:43:21.315433+0000","last_active":"2024-09-21T09:43:26.350506+0000","last_peered":"2024-09-21T09:43:26.350506+0000","last_clean":"2024-09-21T09:43:26.350506+0000","last_became_active":"2024-09-21T09:43:21.315100+0000","last_became_peered":"2024-09-21T09:43:21.315100+0000","last_unstale":"2024-09-21T09:43:26.350506+0000","last_undegraded":"2024-09-21T09:43:26.350506+0000","last_fullsized":"2024-09-21T09:43:26.350506+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:05:55.727692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049819100000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":48,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.349885+0000","last_change":"2024-09-21T09:43:21.315140+0000","last_active":"2024-09-21T09:43:26.349885+0000","last_peered":"2024-09-21T09:43:26.349885+0000","last_clean":"2024-09-21T09:43:26.349885+0000","last_became_active":"2024-09-21T09:43:21.314817+0000","last_became_peered":"2024-09-21T09:43:21.314817+0000","last_unstale":"2024-09-21T09:43:26.349885+0000","last_undegraded":"2024-09-21T09:43:26.349885+0000","last_fullsized":"2024-09-21T09:43:26.349885+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:25.622786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088008499999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"16'1","reported_seq":47,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350078+0000","last_change":"2024-09-21T09:43:21.315433+0000","last_active":"2024-09-21T09:43:26.350078+0000","last_peered":"2024-09-21T09:43:26.350078+0000","last_clean":"2024-09-21T09:43:26.350078+0000","last_became_active":"2024-09-21T09:43:21.315110+0000","last_became_peered":"2024-09-21T09:43:21.315110+0000","last_unstale":"2024-09-21T09:43:26.350078+0000","last_undegraded":"2024-09-21T09:43:26.350078+0000","last_fullsized":"2024-09-21T09:43:26.350078+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:42.627708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010430319999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"21'104","reported_seq":195,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.720214+0000","last_change":"2024-09-21T09:43:19.310088+0000","last_active":"2024-09-21T09:43:26.720214+0000","last_peered":"2024-09-21T09:43:26.720214+0000","last_clean":"2024-09-21T09:43:26.720214+0000","last_became_active":"2024-09-21T09:43:19.309749+0000","last_became_peered":"2024-09-21T09:43:19.309749+0000","last_unstale":"2024-09-21T09:43:26.720214+0000","last_undegraded":"2024-09-21T09:43:26.720214+0000","last_fullsized":"2024-09-21T09:43:26.720214+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:50.808452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:50.808452+0000","last_clean_scrub_stamp":"2024-09-21T09:41:50.808452+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:18:22.448558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":30,"seq":128849018884,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27820,"kb_used_data":276,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344020,"statfs":{"total":96636764160,"available":96608276480,"internally_reserved":0,"allocated":282624,"data_stored":96278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4436,"internal_metadata":28176044},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":29,"seq":124554051588,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27816,"kb_used_data":272,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344024,"statfs":{"total":96636764160,"available":96608280576,"internally_reserved":0,"allocated":278528,"data_stored":96259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4432,"internal_metadata":28176048},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":28,"seq":120259084292,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28336,"kb_used_data":856,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343504,"statfs":{"total":96636764160,"available":96607748096,"internally_reserved":0,"allocated":876544,"data_stored":686646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4431,"internal_metadata":28110513},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":27,"seq":115964116996,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28320,"kb_used_data":836,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343520,"statfs":{"total":96636764160,"available":96607764480,"internally_reserved":0,"allocated":856064,"data_stored":679465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4427,"internal_metadata":28110517},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:43:30.372 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T09:43:30.472 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.470+0000 7fbfb331a640 1 -- 172.21.15.92:0/560167620 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbfac095c20 msgr2=0x7fbfac096080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:30.472 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.470+0000 7fbfb331a640 1 --2- 172.21.15.92:0/560167620 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbfac095c20 0x7fbfac096080 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7fbfa4005fd0 tx=0x7fbfa402a8e0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.472 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.470+0000 7fbfb331a640 1 -- 172.21.15.92:0/560167620 shutdown_connections 2024-09-21T09:43:30.473 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.470+0000 7fbfb331a640 1 --2- 172.21.15.92:0/560167620 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbfac095c20 0x7fbfac096080 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.473 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.470+0000 7fbfb331a640 1 --2- 172.21.15.92:0/560167620 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbfac12c160 0x7fbfac0956e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.473 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.470+0000 7fbfb331a640 1 --2- 172.21.15.92:0/560167620 >> v2:172.21.15.92:3300/0 conn(0x7fbfac0991f0 0x7fbfac12bc20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.473 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.470+0000 7fbfb331a640 1 -- 172.21.15.92:0/560167620 >> 172.21.15.92:0/560167620 conn(0x7fbfac09a2e0 msgr2=0x7fbfac097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:30.473 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.470+0000 7fbfb331a640 1 -- 172.21.15.92:0/560167620 shutdown_connections 2024-09-21T09:43:30.473 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.470+0000 7fbfb331a640 1 -- 172.21.15.92:0/560167620 wait complete. 2024-09-21T09:43:30.475 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.471+0000 7fbfb331a640 1 Processor -- start 2024-09-21T09:43:30.475 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.471+0000 7fbfb331a640 1 -- start start 2024-09-21T09:43:30.475 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.471+0000 7fbfb331a640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbfac095c20 0x7fbfac1c2bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:30.475 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.472+0000 7fbfb331a640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbfac0991f0 0x7fbfac1c3100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:30.475 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.472+0000 7fbfb331a640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbfac12c160 0x7fbfac0713c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:30.476 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.472+0000 7fbfb331a640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fbfac1334e0 con 0x7fbfac12c160 2024-09-21T09:43:30.476 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.472+0000 7fbfb331a640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fbfac133360 con 0x7fbfac095c20 2024-09-21T09:43:30.476 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.472+0000 7fbfb331a640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fbfac133660 con 0x7fbfac0991f0 2024-09-21T09:43:30.476 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.472+0000 7fbfb10b6640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbfac095c20 0x7fbfac1c2bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:30.476 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.472+0000 7fbfb18b7640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbfac12c160 0x7fbfac0713c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:30.476 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.472+0000 7fbfb08b5640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbfac0991f0 0x7fbfac1c3100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:30.476 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.472+0000 7fbfb10b6640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbfac095c20 0x7fbfac1c2bc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:44582/0 (socket says 172.21.15.92:44582) 2024-09-21T09:43:30.476 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.472+0000 7fbfb10b6640 1 -- 172.21.15.92:0/3611351088 learned_addr learned my addr 172.21.15.92:0/3611351088 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:30.476 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.472+0000 7fbfb18b7640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbfac12c160 0x7fbfac0713c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:43018/0 (socket says 172.21.15.92:43018) 2024-09-21T09:43:30.476 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.472+0000 7fbfb08b5640 1 -- 172.21.15.92:0/3611351088 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbfac095c20 msgr2=0x7fbfac1c2bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:30.476 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.472+0000 7fbfb08b5640 1 --2- 172.21.15.92:0/3611351088 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbfac095c20 0x7fbfac1c2bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.476 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.472+0000 7fbfb08b5640 1 -- 172.21.15.92:0/3611351088 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbfac12c160 msgr2=0x7fbfac0713c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:30.476 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.472+0000 7fbfb08b5640 1 --2- 172.21.15.92:0/3611351088 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbfac12c160 0x7fbfac0713c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.476 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.472+0000 7fbfb08b5640 1 -- 172.21.15.92:0/3611351088 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbfac0738c0 con 0x7fbfac0991f0 2024-09-21T09:43:30.477 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.472+0000 7fbfb18b7640 1 --2- 172.21.15.92:0/3611351088 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbfac12c160 0x7fbfac0713c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:30.477 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.472+0000 7fbfb08b5640 1 --2- 172.21.15.92:0/3611351088 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbfac0991f0 0x7fbfac1c3100 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fbf9c00ea20 tx=0x7fbf9c00eee0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:30.477 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.472+0000 7fbfa1ffb640 1 -- 172.21.15.92:0/3611351088 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbf9c00cd50 con 0x7fbfac0991f0 2024-09-21T09:43:30.477 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.472+0000 7fbfa1ffb640 1 -- 172.21.15.92:0/3611351088 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fbf9c004020 con 0x7fbfac0991f0 2024-09-21T09:43:30.477 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.472+0000 7fbfa1ffb640 1 -- 172.21.15.92:0/3611351088 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbf9c004450 con 0x7fbfac0991f0 2024-09-21T09:43:30.477 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.472+0000 7fbfb331a640 1 -- 172.21.15.92:0/3611351088 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbfac12cb40 con 0x7fbfac0991f0 2024-09-21T09:43:30.477 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.473+0000 7fbfb331a640 1 -- 172.21.15.92:0/3611351088 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fbfac1c3640 con 0x7fbfac0991f0 2024-09-21T09:43:30.477 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.473+0000 7fbfa1ffb640 1 -- 172.21.15.92:0/3611351088 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7fbf9c0035f0 con 0x7fbfac0991f0 2024-09-21T09:43:30.477 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.473+0000 7fbfb331a640 1 -- 172.21.15.92:0/3611351088 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbfac12bc20 con 0x7fbfac0991f0 2024-09-21T09:43:30.477 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.474+0000 7fbfa1ffb640 1 -- 172.21.15.92:0/3611351088 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fbf9c069870 con 0x7fbfac0991f0 2024-09-21T09:43:30.479 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.477+0000 7fbfa1ffb640 1 -- 172.21.15.92:0/3611351088 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fbf9c068880 con 0x7fbfac0991f0 2024-09-21T09:43:30.625 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.623+0000 7fbfb331a640 1 -- 172.21.15.92:0/3611351088 --> v1:172.21.15.92:6800/2299163855 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7fbfac0699c0 con 0x7fbf900514e0 2024-09-21T09:43:30.626 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.624+0000 7fbfa1ffb640 1 -- 172.21.15.92:0/3611351088 <== mgr.14102 v1:172.21.15.92:6800/2299163855 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32001 (unknown 2965378022 0 2174376010) 0x7fbfac0699c0 con 0x7fbf900514e0 2024-09-21T09:43:30.626 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:30.626 INFO:teuthology.orchestra.run.smithi092.stderr:dumped all 2024-09-21T09:43:30.627 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.625+0000 7fbfb331a640 1 -- 172.21.15.92:0/3611351088 >> v1:172.21.15.92:6800/2299163855 conn(0x7fbf900514e0 legacy=0x7fbf900539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:30.628 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.626+0000 7fbfb331a640 1 -- 172.21.15.92:0/3611351088 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbfac0991f0 msgr2=0x7fbfac1c3100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:30.628 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.626+0000 7fbfb331a640 1 --2- 172.21.15.92:0/3611351088 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbfac0991f0 0x7fbfac1c3100 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fbf9c00ea20 tx=0x7fbf9c00eee0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.628 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.626+0000 7fbfb331a640 1 -- 172.21.15.92:0/3611351088 shutdown_connections 2024-09-21T09:43:30.628 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.626+0000 7fbfb331a640 1 --2- 172.21.15.92:0/3611351088 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbfac12c160 0x7fbfac0713c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.628 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.626+0000 7fbfb331a640 1 --2- 172.21.15.92:0/3611351088 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbfac0991f0 0x7fbfac1c3100 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.628 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.626+0000 7fbfb331a640 1 --2- 172.21.15.92:0/3611351088 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbfac095c20 0x7fbfac1c2bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.628 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.626+0000 7fbfb331a640 1 -- 172.21.15.92:0/3611351088 >> 172.21.15.92:0/3611351088 conn(0x7fbfac09a2e0 msgr2=0x7fbfac09be70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:30.628 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.626+0000 7fbfb331a640 1 -- 172.21.15.92:0/3611351088 shutdown_connections 2024-09-21T09:43:30.628 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.626+0000 7fbfb331a640 1 -- 172.21.15.92:0/3611351088 wait complete. 2024-09-21T09:43:30.638 INFO:teuthology.orchestra.run.smithi092.stdout:{"pg_ready":true,"pg_map":{"version":27,"stamp":"2024-09-21T09:43:30.275953+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112292,"kb_used_data":2240,"kb_used_omap":17,"kb_used_meta":109934,"kb_avail":377375068,"statfs":{"total":386547056640,"available":386432069632,"internally_reserved":0,"allocated":2293760,"data_stored":1558648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17726,"internal_metadata":112573122},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001446"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":52,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350236+0000","last_change":"2024-09-21T09:43:21.330614+0000","last_active":"2024-09-21T09:43:26.350236+0000","last_peered":"2024-09-21T09:43:26.350236+0000","last_clean":"2024-09-21T09:43:26.350236+0000","last_became_active":"2024-09-21T09:43:21.330325+0000","last_became_peered":"2024-09-21T09:43:21.330325+0000","last_unstale":"2024-09-21T09:43:26.350236+0000","last_undegraded":"2024-09-21T09:43:26.350236+0000","last_fullsized":"2024-09-21T09:43:26.350236+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:17:00.614617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084231599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":42,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.720522+0000","last_change":"2024-09-21T09:43:19.309958+0000","last_active":"2024-09-21T09:43:26.720522+0000","last_peered":"2024-09-21T09:43:26.720522+0000","last_clean":"2024-09-21T09:43:26.720522+0000","last_became_active":"2024-09-21T09:43:19.309603+0000","last_became_peered":"2024-09-21T09:43:19.309603+0000","last_unstale":"2024-09-21T09:43:26.720522+0000","last_undegraded":"2024-09-21T09:43:26.720522+0000","last_fullsized":"2024-09-21T09:43:26.720522+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:26:08.037398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049204800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":52,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350258+0000","last_change":"2024-09-21T09:43:21.330875+0000","last_active":"2024-09-21T09:43:26.350258+0000","last_peered":"2024-09-21T09:43:26.350258+0000","last_clean":"2024-09-21T09:43:26.350258+0000","last_became_active":"2024-09-21T09:43:21.330619+0000","last_became_peered":"2024-09-21T09:43:21.330619+0000","last_unstale":"2024-09-21T09:43:26.350258+0000","last_undegraded":"2024-09-21T09:43:26.350258+0000","last_fullsized":"2024-09-21T09:43:26.350258+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:22:03.959366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080402800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":42,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.720283+0000","last_change":"2024-09-21T09:43:19.310001+0000","last_active":"2024-09-21T09:43:26.720283+0000","last_peered":"2024-09-21T09:43:26.720283+0000","last_clean":"2024-09-21T09:43:26.720283+0000","last_became_active":"2024-09-21T09:43:19.309659+0000","last_became_peered":"2024-09-21T09:43:19.309659+0000","last_unstale":"2024-09-21T09:43:26.720283+0000","last_undegraded":"2024-09-21T09:43:26.720283+0000","last_fullsized":"2024-09-21T09:43:26.720283+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:39.861259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000465849,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"18'2","reported_seq":50,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350048+0000","last_change":"2024-09-21T09:43:21.315323+0000","last_active":"2024-09-21T09:43:26.350048+0000","last_peered":"2024-09-21T09:43:26.350048+0000","last_clean":"2024-09-21T09:43:26.350048+0000","last_became_active":"2024-09-21T09:43:21.314821+0000","last_became_peered":"2024-09-21T09:43:21.314821+0000","last_unstale":"2024-09-21T09:43:26.350048+0000","last_undegraded":"2024-09-21T09:43:26.350048+0000","last_fullsized":"2024-09-21T09:43:26.350048+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:32.540545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010637209999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":39,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350506+0000","last_change":"2024-09-21T09:43:21.315433+0000","last_active":"2024-09-21T09:43:26.350506+0000","last_peered":"2024-09-21T09:43:26.350506+0000","last_clean":"2024-09-21T09:43:26.350506+0000","last_became_active":"2024-09-21T09:43:21.315100+0000","last_became_peered":"2024-09-21T09:43:21.315100+0000","last_unstale":"2024-09-21T09:43:26.350506+0000","last_undegraded":"2024-09-21T09:43:26.350506+0000","last_fullsized":"2024-09-21T09:43:26.350506+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:05:55.727692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049819100000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":48,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.349885+0000","last_change":"2024-09-21T09:43:21.315140+0000","last_active":"2024-09-21T09:43:26.349885+0000","last_peered":"2024-09-21T09:43:26.349885+0000","last_clean":"2024-09-21T09:43:26.349885+0000","last_became_active":"2024-09-21T09:43:21.314817+0000","last_became_peered":"2024-09-21T09:43:21.314817+0000","last_unstale":"2024-09-21T09:43:26.349885+0000","last_undegraded":"2024-09-21T09:43:26.349885+0000","last_fullsized":"2024-09-21T09:43:26.349885+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:25.622786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088008499999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"16'1","reported_seq":47,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350078+0000","last_change":"2024-09-21T09:43:21.315433+0000","last_active":"2024-09-21T09:43:26.350078+0000","last_peered":"2024-09-21T09:43:26.350078+0000","last_clean":"2024-09-21T09:43:26.350078+0000","last_became_active":"2024-09-21T09:43:21.315110+0000","last_became_peered":"2024-09-21T09:43:21.315110+0000","last_unstale":"2024-09-21T09:43:26.350078+0000","last_undegraded":"2024-09-21T09:43:26.350078+0000","last_fullsized":"2024-09-21T09:43:26.350078+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:42.627708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010430319999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"21'104","reported_seq":195,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.720214+0000","last_change":"2024-09-21T09:43:19.310088+0000","last_active":"2024-09-21T09:43:26.720214+0000","last_peered":"2024-09-21T09:43:26.720214+0000","last_clean":"2024-09-21T09:43:26.720214+0000","last_became_active":"2024-09-21T09:43:19.309749+0000","last_became_peered":"2024-09-21T09:43:19.309749+0000","last_unstale":"2024-09-21T09:43:26.720214+0000","last_undegraded":"2024-09-21T09:43:26.720214+0000","last_fullsized":"2024-09-21T09:43:26.720214+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:50.808452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:50.808452+0000","last_clean_scrub_stamp":"2024-09-21T09:41:50.808452+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:18:22.448558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":30,"seq":128849018884,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27820,"kb_used_data":276,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344020,"statfs":{"total":96636764160,"available":96608276480,"internally_reserved":0,"allocated":282624,"data_stored":96278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4436,"internal_metadata":28176044},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":29,"seq":124554051588,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27816,"kb_used_data":272,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344024,"statfs":{"total":96636764160,"available":96608280576,"internally_reserved":0,"allocated":278528,"data_stored":96259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4432,"internal_metadata":28176048},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":28,"seq":120259084292,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28336,"kb_used_data":856,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343504,"statfs":{"total":96636764160,"available":96607748096,"internally_reserved":0,"allocated":876544,"data_stored":686646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4431,"internal_metadata":28110513},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":27,"seq":115964116996,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28320,"kb_used_data":836,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343520,"statfs":{"total":96636764160,"available":96607764480,"internally_reserved":0,"allocated":856064,"data_stored":679465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4427,"internal_metadata":28110517},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:43:30.639 INFO:tasks.ceph.ceph_manager.ceph:clean! 2024-09-21T09:43:30.639 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T09:43:30.738 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.736+0000 7f65d61a5640 1 -- 172.21.15.92:0/3881774535 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f65d0095df0 msgr2=0x7f65d0133b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:30.738 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.736+0000 7f65d61a5640 1 --2- 172.21.15.92:0/3881774535 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f65d0095df0 0x7f65d0133b10 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f65c8005fd0 tx=0x7f65c802a980 comp rx=0 tx=0).stop 2024-09-21T09:43:30.738 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.736+0000 7f65d61a5640 1 -- 172.21.15.92:0/3881774535 shutdown_connections 2024-09-21T09:43:30.738 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.736+0000 7f65d61a5640 1 --2- 172.21.15.92:0/3881774535 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f65d0095df0 0x7f65d0133b10 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.739 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.736+0000 7f65d61a5640 1 --2- 172.21.15.92:0/3881774535 >> v2:172.21.15.92:3300/0 conn(0x7f65d00991f0 0x7f65d012bc20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.739 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.736+0000 7f65d61a5640 1 -- 172.21.15.92:0/3881774535 >> 172.21.15.92:0/3881774535 conn(0x7f65d009a2e0 msgr2=0x7f65d0097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:30.739 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.736+0000 7f65d61a5640 1 -- 172.21.15.92:0/3881774535 shutdown_connections 2024-09-21T09:43:30.739 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.736+0000 7f65d61a5640 1 -- 172.21.15.92:0/3881774535 wait complete. 2024-09-21T09:43:30.741 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.737+0000 7f65d61a5640 1 Processor -- start 2024-09-21T09:43:30.741 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.737+0000 7f65d61a5640 1 -- start start 2024-09-21T09:43:30.741 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.737+0000 7f65d61a5640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f65d0095df0 0x7f65d0072b90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:30.742 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.737+0000 7f65d61a5640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f65d00991f0 0x7f65d00730d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:30.742 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.737+0000 7f65d61a5640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f65d012c160 0x7f65d0073610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:30.742 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.737+0000 7f65d61a5640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f65d01346e0 con 0x7f65d00991f0 2024-09-21T09:43:30.742 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.737+0000 7f65d61a5640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f65d0134560 con 0x7f65d0095df0 2024-09-21T09:43:30.742 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.737+0000 7f65d61a5640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f65d0134860 con 0x7f65d012c160 2024-09-21T09:43:30.742 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.737+0000 7f65cf7fe640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f65d0095df0 0x7f65d0072b90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:30.742 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.737+0000 7f65c7fff640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f65d00991f0 0x7f65d00730d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:30.742 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.737+0000 7f65cf7fe640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f65d0095df0 0x7f65d0072b90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:44612/0 (socket says 172.21.15.92:44612) 2024-09-21T09:43:30.742 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.737+0000 7f65cf7fe640 1 -- 172.21.15.92:0/1874904533 learned_addr learned my addr 172.21.15.92:0/1874904533 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:30.742 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.737+0000 7f65cffff640 1 --2- 172.21.15.92:0/1874904533 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f65d012c160 0x7f65d0073610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:30.742 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.738+0000 7f65cf7fe640 1 -- 172.21.15.92:0/1874904533 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f65d012c160 msgr2=0x7f65d0073610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:30.742 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.738+0000 7f65cf7fe640 1 --2- 172.21.15.92:0/1874904533 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f65d012c160 0x7f65d0073610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.742 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.738+0000 7f65cf7fe640 1 -- 172.21.15.92:0/1874904533 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f65d00991f0 msgr2=0x7f65d00730d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:30.742 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.738+0000 7f65cf7fe640 1 --2- 172.21.15.92:0/1874904533 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f65d00991f0 0x7f65d00730d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.738+0000 7f65cf7fe640 1 -- 172.21.15.92:0/1874904533 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f65d01226c0 con 0x7f65d0095df0 2024-09-21T09:43:30.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.738+0000 7f65c7fff640 1 --2- 172.21.15.92:0/1874904533 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f65d00991f0 0x7f65d00730d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:30.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.738+0000 7f65cf7fe640 1 --2- 172.21.15.92:0/1874904533 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f65d0095df0 0x7f65d0072b90 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f65c000ea90 tx=0x7f65c000ef50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:30.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.738+0000 7f65ccff9640 1 -- 172.21.15.92:0/1874904533 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f65c000ce40 con 0x7f65d0095df0 2024-09-21T09:43:30.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.738+0000 7f65ccff9640 1 -- 172.21.15.92:0/1874904533 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f65c0004020 con 0x7f65d0095df0 2024-09-21T09:43:30.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.738+0000 7f65d61a5640 1 -- 172.21.15.92:0/1874904533 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f65d0094030 con 0x7f65d0095df0 2024-09-21T09:43:30.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.738+0000 7f65ccff9640 1 -- 172.21.15.92:0/1874904533 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f65c0004470 con 0x7f65d0095df0 2024-09-21T09:43:30.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.738+0000 7f65d61a5640 1 -- 172.21.15.92:0/1874904533 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f65d0094840 con 0x7f65d0095df0 2024-09-21T09:43:30.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.739+0000 7f65d61a5640 1 -- 172.21.15.92:0/1874904533 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f65d0091bc0 con 0x7f65d0095df0 2024-09-21T09:43:30.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.739+0000 7f65ccff9640 1 -- 172.21.15.92:0/1874904533 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f65c00035f0 con 0x7f65d0095df0 2024-09-21T09:43:30.743 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.740+0000 7f65ccff9640 1 -- 172.21.15.92:0/1874904533 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f65c0069940 con 0x7f65d0095df0 2024-09-21T09:43:30.745 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.743+0000 7f65ccff9640 1 -- 172.21.15.92:0/1874904533 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f65c00689f0 con 0x7f65d0095df0 2024-09-21T09:43:30.889 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.887+0000 7f65d61a5640 1 -- 172.21.15.92:0/1874904533 --> v1:172.21.15.92:6800/2299163855 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f65d012bc20 con 0x7f65b00514b0 2024-09-21T09:43:30.889 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.888+0000 7f65ccff9640 1 -- 172.21.15.92:0/1874904533 <== mgr.14102 v1:172.21.15.92:6800/2299163855 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32001 (unknown 2965378022 0 2174376010) 0x7f65d012bc20 con 0x7f65b00514b0 2024-09-21T09:43:30.890 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:30.890 INFO:teuthology.orchestra.run.smithi092.stderr:dumped all 2024-09-21T09:43:30.891 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.889+0000 7f65d61a5640 1 -- 172.21.15.92:0/1874904533 >> v1:172.21.15.92:6800/2299163855 conn(0x7f65b00514b0 legacy=0x7f65b0053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:30.892 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.889+0000 7f65d61a5640 1 -- 172.21.15.92:0/1874904533 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f65d0095df0 msgr2=0x7f65d0072b90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:30.892 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.889+0000 7f65d61a5640 1 --2- 172.21.15.92:0/1874904533 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f65d0095df0 0x7f65d0072b90 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f65c000ea90 tx=0x7f65c000ef50 comp rx=0 tx=0).stop 2024-09-21T09:43:30.892 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.889+0000 7f65d61a5640 1 -- 172.21.15.92:0/1874904533 shutdown_connections 2024-09-21T09:43:30.892 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.889+0000 7f65d61a5640 1 --2- 172.21.15.92:0/1874904533 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f65d012c160 0x7f65d0073610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.892 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.889+0000 7f65d61a5640 1 --2- 172.21.15.92:0/1874904533 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f65d00991f0 0x7f65d00730d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.892 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.889+0000 7f65d61a5640 1 --2- 172.21.15.92:0/1874904533 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f65d0095df0 0x7f65d0072b90 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:30.892 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.889+0000 7f65d61a5640 1 -- 172.21.15.92:0/1874904533 >> 172.21.15.92:0/1874904533 conn(0x7f65d009a2e0 msgr2=0x7f65d008bf80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:30.892 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.890+0000 7f65d61a5640 1 -- 172.21.15.92:0/1874904533 shutdown_connections 2024-09-21T09:43:30.892 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.890+0000 7f65d61a5640 1 -- 172.21.15.92:0/1874904533 wait complete. 2024-09-21T09:43:30.902 INFO:teuthology.orchestra.run.smithi092.stdout:{"pg_ready":true,"pg_map":{"version":27,"stamp":"2024-09-21T09:43:30.275953+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112292,"kb_used_data":2240,"kb_used_omap":17,"kb_used_meta":109934,"kb_avail":377375068,"statfs":{"total":386547056640,"available":386432069632,"internally_reserved":0,"allocated":2293760,"data_stored":1558648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17726,"internal_metadata":112573122},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001446"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":52,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350236+0000","last_change":"2024-09-21T09:43:21.330614+0000","last_active":"2024-09-21T09:43:26.350236+0000","last_peered":"2024-09-21T09:43:26.350236+0000","last_clean":"2024-09-21T09:43:26.350236+0000","last_became_active":"2024-09-21T09:43:21.330325+0000","last_became_peered":"2024-09-21T09:43:21.330325+0000","last_unstale":"2024-09-21T09:43:26.350236+0000","last_undegraded":"2024-09-21T09:43:26.350236+0000","last_fullsized":"2024-09-21T09:43:26.350236+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:17:00.614617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084231599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":42,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.720522+0000","last_change":"2024-09-21T09:43:19.309958+0000","last_active":"2024-09-21T09:43:26.720522+0000","last_peered":"2024-09-21T09:43:26.720522+0000","last_clean":"2024-09-21T09:43:26.720522+0000","last_became_active":"2024-09-21T09:43:19.309603+0000","last_became_peered":"2024-09-21T09:43:19.309603+0000","last_unstale":"2024-09-21T09:43:26.720522+0000","last_undegraded":"2024-09-21T09:43:26.720522+0000","last_fullsized":"2024-09-21T09:43:26.720522+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:26:08.037398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049204800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":52,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350258+0000","last_change":"2024-09-21T09:43:21.330875+0000","last_active":"2024-09-21T09:43:26.350258+0000","last_peered":"2024-09-21T09:43:26.350258+0000","last_clean":"2024-09-21T09:43:26.350258+0000","last_became_active":"2024-09-21T09:43:21.330619+0000","last_became_peered":"2024-09-21T09:43:21.330619+0000","last_unstale":"2024-09-21T09:43:26.350258+0000","last_undegraded":"2024-09-21T09:43:26.350258+0000","last_fullsized":"2024-09-21T09:43:26.350258+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:22:03.959366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080402800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":42,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.720283+0000","last_change":"2024-09-21T09:43:19.310001+0000","last_active":"2024-09-21T09:43:26.720283+0000","last_peered":"2024-09-21T09:43:26.720283+0000","last_clean":"2024-09-21T09:43:26.720283+0000","last_became_active":"2024-09-21T09:43:19.309659+0000","last_became_peered":"2024-09-21T09:43:19.309659+0000","last_unstale":"2024-09-21T09:43:26.720283+0000","last_undegraded":"2024-09-21T09:43:26.720283+0000","last_fullsized":"2024-09-21T09:43:26.720283+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:39.861259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000465849,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"18'2","reported_seq":50,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350048+0000","last_change":"2024-09-21T09:43:21.315323+0000","last_active":"2024-09-21T09:43:26.350048+0000","last_peered":"2024-09-21T09:43:26.350048+0000","last_clean":"2024-09-21T09:43:26.350048+0000","last_became_active":"2024-09-21T09:43:21.314821+0000","last_became_peered":"2024-09-21T09:43:21.314821+0000","last_unstale":"2024-09-21T09:43:26.350048+0000","last_undegraded":"2024-09-21T09:43:26.350048+0000","last_fullsized":"2024-09-21T09:43:26.350048+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:32.540545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010637209999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":39,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350506+0000","last_change":"2024-09-21T09:43:21.315433+0000","last_active":"2024-09-21T09:43:26.350506+0000","last_peered":"2024-09-21T09:43:26.350506+0000","last_clean":"2024-09-21T09:43:26.350506+0000","last_became_active":"2024-09-21T09:43:21.315100+0000","last_became_peered":"2024-09-21T09:43:21.315100+0000","last_unstale":"2024-09-21T09:43:26.350506+0000","last_undegraded":"2024-09-21T09:43:26.350506+0000","last_fullsized":"2024-09-21T09:43:26.350506+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:05:55.727692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049819100000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":48,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.349885+0000","last_change":"2024-09-21T09:43:21.315140+0000","last_active":"2024-09-21T09:43:26.349885+0000","last_peered":"2024-09-21T09:43:26.349885+0000","last_clean":"2024-09-21T09:43:26.349885+0000","last_became_active":"2024-09-21T09:43:21.314817+0000","last_became_peered":"2024-09-21T09:43:21.314817+0000","last_unstale":"2024-09-21T09:43:26.349885+0000","last_undegraded":"2024-09-21T09:43:26.349885+0000","last_fullsized":"2024-09-21T09:43:26.349885+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:25.622786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088008499999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"16'1","reported_seq":47,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350078+0000","last_change":"2024-09-21T09:43:21.315433+0000","last_active":"2024-09-21T09:43:26.350078+0000","last_peered":"2024-09-21T09:43:26.350078+0000","last_clean":"2024-09-21T09:43:26.350078+0000","last_became_active":"2024-09-21T09:43:21.315110+0000","last_became_peered":"2024-09-21T09:43:21.315110+0000","last_unstale":"2024-09-21T09:43:26.350078+0000","last_undegraded":"2024-09-21T09:43:26.350078+0000","last_fullsized":"2024-09-21T09:43:26.350078+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:42.627708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010430319999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"21'104","reported_seq":195,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.720214+0000","last_change":"2024-09-21T09:43:19.310088+0000","last_active":"2024-09-21T09:43:26.720214+0000","last_peered":"2024-09-21T09:43:26.720214+0000","last_clean":"2024-09-21T09:43:26.720214+0000","last_became_active":"2024-09-21T09:43:19.309749+0000","last_became_peered":"2024-09-21T09:43:19.309749+0000","last_unstale":"2024-09-21T09:43:26.720214+0000","last_undegraded":"2024-09-21T09:43:26.720214+0000","last_fullsized":"2024-09-21T09:43:26.720214+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:50.808452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:50.808452+0000","last_clean_scrub_stamp":"2024-09-21T09:41:50.808452+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:18:22.448558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":30,"seq":128849018884,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27820,"kb_used_data":276,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344020,"statfs":{"total":96636764160,"available":96608276480,"internally_reserved":0,"allocated":282624,"data_stored":96278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4436,"internal_metadata":28176044},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":29,"seq":124554051588,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27816,"kb_used_data":272,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344024,"statfs":{"total":96636764160,"available":96608280576,"internally_reserved":0,"allocated":278528,"data_stored":96259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4432,"internal_metadata":28176048},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":28,"seq":120259084292,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28336,"kb_used_data":856,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343504,"statfs":{"total":96636764160,"available":96607748096,"internally_reserved":0,"allocated":876544,"data_stored":686646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4431,"internal_metadata":28110513},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":27,"seq":115964116996,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28320,"kb_used_data":836,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343520,"statfs":{"total":96636764160,"available":96607764480,"internally_reserved":0,"allocated":856064,"data_stored":679465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4427,"internal_metadata":28110517},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:43:30.902 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T09:43:31.001 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.999+0000 7f95ac4a5640 1 -- 172.21.15.92:0/1899131510 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f95a412c200 msgr2=0x7f95a40956e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:31.001 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.999+0000 7f95ac4a5640 1 --2- 172.21.15.92:0/1899131510 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f95a412c200 0x7f95a40956e0 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f9598009900 tx=0x7f959802a460 comp rx=0 tx=0).stop 2024-09-21T09:43:31.001 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.999+0000 7f95ac4a5640 1 -- 172.21.15.92:0/1899131510 shutdown_connections 2024-09-21T09:43:31.001 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.999+0000 7f95ac4a5640 1 --2- 172.21.15.92:0/1899131510 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f95a4095c20 0x7f95a4096070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:31.001 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.999+0000 7f95ac4a5640 1 --2- 172.21.15.92:0/1899131510 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f95a412c200 0x7f95a40956e0 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:31.001 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.999+0000 7f95ac4a5640 1 --2- 172.21.15.92:0/1899131510 >> v2:172.21.15.92:3300/0 conn(0x7f95a40991f0 0x7f95a412bcc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:31.002 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.999+0000 7f95ac4a5640 1 -- 172.21.15.92:0/1899131510 >> 172.21.15.92:0/1899131510 conn(0x7f95a409a2e0 msgr2=0x7f95a4097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:31.002 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.999+0000 7f95ac4a5640 1 -- 172.21.15.92:0/1899131510 shutdown_connections 2024-09-21T09:43:31.002 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:30.999+0000 7f95ac4a5640 1 -- 172.21.15.92:0/1899131510 wait complete. 2024-09-21T09:43:31.004 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.000+0000 7f95ac4a5640 1 Processor -- start 2024-09-21T09:43:31.004 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.000+0000 7f95ac4a5640 1 -- start start 2024-09-21T09:43:31.004 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.000+0000 7f95ac4a5640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f95a4095c20 0x7f95a41c2840 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:31.004 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.000+0000 7f95ac4a5640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f95a40991f0 0x7f95a41c2d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:31.004 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.001+0000 7f95ac4a5640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f95a412c200 0x7f95a41c32c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:31.004 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.001+0000 7f95ac4a5640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f95a41333d0 con 0x7f95a4095c20 2024-09-21T09:43:31.004 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.001+0000 7f95ac4a5640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f95a4133250 con 0x7f95a412c200 2024-09-21T09:43:31.004 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.001+0000 7f95ac4a5640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f95a4133550 con 0x7f95a40991f0 2024-09-21T09:43:31.004 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.001+0000 7f95aa241640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f95a4095c20 0x7f95a41c2840 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:31.004 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.001+0000 7f95aa241640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f95a4095c20 0x7f95a41c2840 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:43064/0 (socket says 172.21.15.92:43064) 2024-09-21T09:43:31.005 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.001+0000 7f95aa241640 1 -- 172.21.15.92:0/439507898 learned_addr learned my addr 172.21.15.92:0/439507898 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:31.005 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.001+0000 7f95aaa42640 1 --2- 172.21.15.92:0/439507898 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f95a412c200 0x7f95a41c32c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:31.005 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.001+0000 7f95a9a40640 1 --2- 172.21.15.92:0/439507898 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f95a40991f0 0x7f95a41c2d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:31.005 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.001+0000 7f95aa241640 1 -- 172.21.15.92:0/439507898 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f95a40991f0 msgr2=0x7f95a41c2d80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:31.005 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.001+0000 7f95aa241640 1 --2- 172.21.15.92:0/439507898 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f95a40991f0 0x7f95a41c2d80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:31.005 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.001+0000 7f95aa241640 1 -- 172.21.15.92:0/439507898 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f95a412c200 msgr2=0x7f95a41c32c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:31.005 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.001+0000 7f95aa241640 1 --2- 172.21.15.92:0/439507898 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f95a412c200 0x7f95a41c32c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:31.005 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.001+0000 7f95aa241640 1 -- 172.21.15.92:0/439507898 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f95a4094870 con 0x7f95a4095c20 2024-09-21T09:43:31.005 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.001+0000 7f95aa241640 1 --2- 172.21.15.92:0/439507898 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f95a4095c20 0x7f95a41c2840 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f95a000ea60 tx=0x7f95a000ef20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:31.005 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.001+0000 7f9596ffd640 1 -- 172.21.15.92:0/439507898 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f95a000cde0 con 0x7f95a4095c20 2024-09-21T09:43:31.005 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.001+0000 7f9596ffd640 1 -- 172.21.15.92:0/439507898 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f95a0004020 con 0x7f95a4095c20 2024-09-21T09:43:31.005 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.001+0000 7f9596ffd640 1 -- 172.21.15.92:0/439507898 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f95a0004450 con 0x7f95a4095c20 2024-09-21T09:43:31.005 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.001+0000 7f95ac4a5640 1 -- 172.21.15.92:0/439507898 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f95a406eae0 con 0x7f95a4095c20 2024-09-21T09:43:31.005 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.001+0000 7f95ac4a5640 1 -- 172.21.15.92:0/439507898 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f95a4130830 con 0x7f95a4095c20 2024-09-21T09:43:31.005 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.002+0000 7f95ac4a5640 1 -- 172.21.15.92:0/439507898 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f95a412bcc0 con 0x7f95a4095c20 2024-09-21T09:43:31.006 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.002+0000 7f9596ffd640 1 -- 172.21.15.92:0/439507898 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f95a00035f0 con 0x7f95a4095c20 2024-09-21T09:43:31.006 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.003+0000 7f9596ffd640 1 -- 172.21.15.92:0/439507898 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f95a0069900 con 0x7f95a4095c20 2024-09-21T09:43:31.008 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.006+0000 7f9596ffd640 1 -- 172.21.15.92:0/439507898 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f95a00689b0 con 0x7f95a4095c20 2024-09-21T09:43:31.158 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.157+0000 7f95ac4a5640 1 -- 172.21.15.92:0/439507898 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f95a4131f90 con 0x7f95a4095c20 2024-09-21T09:43:31.159 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.157+0000 7f9596ffd640 1 -- 172.21.15.92:0/439507898 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v33) ==== 74+0+9145 (secure 0 0 0) 0x7f95a00684f0 con 0x7f95a4095c20 2024-09-21T09:43:31.159 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:31.159 INFO:teuthology.orchestra.run.smithi092.stdout:{"epoch":33,"fsid":"5f250e6c-4e36-49b1-ab62-d9454369fa03","created":"2024-09-21T09:41:34.680338+0000","modified":"2024-09-21T09:43:26.338713+0000","last_up_change":"2024-09-21T09:43:20.303761+0000","last_in_change":"2024-09-21T09:41:37.598778+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":9,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"squid","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:41:50.040313+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"32","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T09:41:52.054798+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":18,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f3314f59-3374-4072-85c9-f8c4e458efc8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":21,"up_from":27,"up_thru":27,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6805","nonce":7087154}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6806","nonce":7087154}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6808","nonce":7087154}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6807","nonce":7087154}]},"public_addr":"172.21.15.92:6805/7087154","cluster_addr":"172.21.15.92:6806/7087154","heartbeat_back_addr":"172.21.15.92:6808/7087154","heartbeat_front_addr":"172.21.15.92:6807/7087154","state":["exists","up"]},{"osd":1,"uuid":"cf0486af-ba68-4763-a364-c1f2181604a7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":21,"up_from":28,"up_thru":28,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6801","nonce":2577383826}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6802","nonce":2577383826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6804","nonce":2577383826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6803","nonce":2577383826}]},"public_addr":"172.21.15.92:6801/2577383826","cluster_addr":"172.21.15.92:6802/2577383826","heartbeat_back_addr":"172.21.15.92:6804/2577383826","heartbeat_front_addr":"172.21.15.92:6803/2577383826","state":["exists","up"]},{"osd":2,"uuid":"5c587421-3409-4469-a2ee-b2ea20cd45e5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":11,"last_clean_end":23,"up_from":29,"up_thru":30,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6809","nonce":4235634095}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6810","nonce":4235634095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6812","nonce":4235634095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6811","nonce":4235634095}]},"public_addr":"172.21.15.92:6809/4235634095","cluster_addr":"172.21.15.92:6810/4235634095","heartbeat_back_addr":"172.21.15.92:6812/4235634095","heartbeat_front_addr":"172.21.15.92:6811/4235634095","state":["exists","up"]},{"osd":3,"uuid":"b98663b7-de4d-4c12-ac91-8c33f159da35","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":24,"up_from":30,"up_thru":30,"down_at":25,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6813","nonce":138232475}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6814","nonce":138232475}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6816","nonce":138232475}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.92:6815","nonce":138232475}]},"public_addr":"172.21.15.92:6813/138232475","cluster_addr":"172.21.15.92:6814/138232475","heartbeat_back_addr":"172.21.15.92:6816/138232475","heartbeat_front_addr":"172.21.15.92:6815/138232475","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.934317+0000","dead_epoch":22},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:45.144020+0000","dead_epoch":23},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.895848+0000","dead_epoch":24},{"osd":3,"down_stamp":"2024-09-21T09:43:13.277270+0000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:41:44.636219+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.92:0/230938560":"2024-09-22T09:43:06.249869+0000","172.21.15.92:0/162460762":"2024-09-22T09:43:06.249869+0000","172.21.15.92:6800/1280404317":"2024-09-22T09:43:06.249869+0000","172.21.15.92:0/3587832392":"2024-09-22T09:43:06.249869+0000","172.21.15.92:0/2147030555":"2024-09-22T09:43:06.249869+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:43:31.161 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.159+0000 7f95ac4a5640 1 -- 172.21.15.92:0/439507898 >> v1:172.21.15.92:6800/2299163855 conn(0x7f95880514b0 legacy=0x7f9588053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:31.161 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.159+0000 7f95ac4a5640 1 -- 172.21.15.92:0/439507898 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f95a4095c20 msgr2=0x7f95a41c2840 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:31.161 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.159+0000 7f95ac4a5640 1 --2- 172.21.15.92:0/439507898 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f95a4095c20 0x7f95a41c2840 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f95a000ea60 tx=0x7f95a000ef20 comp rx=0 tx=0).stop 2024-09-21T09:43:31.161 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.159+0000 7f95ac4a5640 1 -- 172.21.15.92:0/439507898 shutdown_connections 2024-09-21T09:43:31.161 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.159+0000 7f95ac4a5640 1 --2- 172.21.15.92:0/439507898 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f95a412c200 0x7f95a41c32c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:31.161 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.159+0000 7f95ac4a5640 1 --2- 172.21.15.92:0/439507898 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f95a40991f0 0x7f95a41c2d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:31.161 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.159+0000 7f95ac4a5640 1 --2- 172.21.15.92:0/439507898 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f95a4095c20 0x7f95a41c2840 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:31.162 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.159+0000 7f95ac4a5640 1 -- 172.21.15.92:0/439507898 >> 172.21.15.92:0/439507898 conn(0x7f95a409a2e0 msgr2=0x7f95a408bf80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:31.162 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.159+0000 7f95ac4a5640 1 -- 172.21.15.92:0/439507898 shutdown_connections 2024-09-21T09:43:31.162 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:31.159+0000 7f95ac4a5640 1 -- 172.21.15.92:0/439507898 wait complete. 2024-09-21T09:43:32.173 INFO:tasks.ceph:Scrubbing osd.0 2024-09-21T09:43:32.174 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 config set osd_debug_deep_scrub_sleep 0 2024-09-21T09:43:32.288 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.286+0000 7f473575f640 1 -- 172.21.15.92:0/2301146113 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f473012bc10 msgr2=0x7f473012bff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:32.289 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.286+0000 7f473575f640 1 --2- 172.21.15.92:0/2301146113 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f473012bc10 0x7f473012bff0 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f4720009860 tx=0x7f472002a3d0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.289 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.286+0000 7f473575f640 1 -- 172.21.15.92:0/2301146113 shutdown_connections 2024-09-21T09:43:32.289 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.286+0000 7f473575f640 1 --2- 172.21.15.92:0/2301146113 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4730096330 0x7f4730133ad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.289 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.286+0000 7f473575f640 1 --2- 172.21.15.92:0/2301146113 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f473012bc10 0x7f473012bff0 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.289 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.286+0000 7f473575f640 1 -- 172.21.15.92:0/2301146113 >> 172.21.15.92:0/2301146113 conn(0x7f473009a3f0 msgr2=0x7f4730097bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:32.289 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.286+0000 7f473575f640 1 -- 172.21.15.92:0/2301146113 shutdown_connections 2024-09-21T09:43:32.289 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.286+0000 7f473575f640 1 -- 172.21.15.92:0/2301146113 wait complete. 2024-09-21T09:43:32.291 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.287+0000 7f473575f640 1 Processor -- start 2024-09-21T09:43:32.291 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.287+0000 7f473575f640 1 -- start start 2024-09-21T09:43:32.292 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.288+0000 7f473575f640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f47300957e0 0x7f47301266a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:32.292 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.288+0000 7f473575f640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4730096330 0x7f4730126be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:32.292 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.288+0000 7f473575f640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f473012bc10 0x7f4730127120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:32.292 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.288+0000 7f473575f640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f4730134790 con 0x7f473012bc10 2024-09-21T09:43:32.292 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.288+0000 7f473575f640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f4730134610 con 0x7f4730096330 2024-09-21T09:43:32.292 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.288+0000 7f473575f640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f4730134910 con 0x7f47300957e0 2024-09-21T09:43:32.292 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.288+0000 7f472effd640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f47300957e0 0x7f47301266a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:32.292 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.288+0000 7f472effd640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f47300957e0 0x7f47301266a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:48410/0 (socket says 172.21.15.92:48410) 2024-09-21T09:43:32.292 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.288+0000 7f472effd640 1 -- 172.21.15.92:0/4003762884 learned_addr learned my addr 172.21.15.92:0/4003762884 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:32.292 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.288+0000 7f472f7fe640 1 --2- 172.21.15.92:0/4003762884 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f473012bc10 0x7f4730127120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:32.292 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.288+0000 7f472e7fc640 1 --2- 172.21.15.92:0/4003762884 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4730096330 0x7f4730126be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:32.292 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.288+0000 7f472effd640 1 -- 172.21.15.92:0/4003762884 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4730096330 msgr2=0x7f4730126be0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:32.292 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.288+0000 7f472effd640 1 --2- 172.21.15.92:0/4003762884 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4730096330 0x7f4730126be0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.293 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.288+0000 7f472effd640 1 -- 172.21.15.92:0/4003762884 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f473012bc10 msgr2=0x7f4730127120 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:32.293 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.288+0000 7f472effd640 1 --2- 172.21.15.92:0/4003762884 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f473012bc10 0x7f4730127120 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.293 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.288+0000 7f472effd640 1 -- 172.21.15.92:0/4003762884 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4730094940 con 0x7f47300957e0 2024-09-21T09:43:32.293 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.288+0000 7f472f7fe640 1 --2- 172.21.15.92:0/4003762884 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f473012bc10 0x7f4730127120 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:32.293 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.288+0000 7f472effd640 1 --2- 172.21.15.92:0/4003762884 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f47300957e0 0x7f47301266a0 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f4720009780 tx=0x7f4720032b90 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:32.293 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.289+0000 7f47177fe640 1 -- 172.21.15.92:0/4003762884 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4720040070 con 0x7f47300957e0 2024-09-21T09:43:32.293 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.289+0000 7f47177fe640 1 -- 172.21.15.92:0/4003762884 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f4720005330 con 0x7f47300957e0 2024-09-21T09:43:32.293 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.289+0000 7f473575f640 1 -- 172.21.15.92:0/4003762884 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4730093fa0 con 0x7f47300957e0 2024-09-21T09:43:32.293 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.289+0000 7f47177fe640 1 -- 172.21.15.92:0/4003762884 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f47200047a0 con 0x7f47300957e0 2024-09-21T09:43:32.293 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.289+0000 7f473575f640 1 -- 172.21.15.92:0/4003762884 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f4730094f60 con 0x7f47300957e0 2024-09-21T09:43:32.293 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.289+0000 7f473575f640 1 -- 172.21.15.92:0/4003762884 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f473012bff0 con 0x7f47300957e0 2024-09-21T09:43:32.293 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.290+0000 7f47177fe640 1 -- 172.21.15.92:0/4003762884 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f4720039040 con 0x7f47300957e0 2024-09-21T09:43:32.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.290+0000 7f47177fe640 1 -- 172.21.15.92:0/4003762884 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f4720089ee0 con 0x7f47300957e0 2024-09-21T09:43:32.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.290+0000 7f47177fe640 1 -- 172.21.15.92:0/4003762884 --> v1:172.21.15.92:6805/7087154 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f471c05b9e0 con 0x7f471c057d30 2024-09-21T09:43:32.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.290+0000 7f47177fe640 1 -- 172.21.15.92:0/4003762884 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7f472008c5e0 con 0x7f47300957e0 2024-09-21T09:43:32.294 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.291+0000 7f47177fe640 1 -- 172.21.15.92:0/4003762884 <== osd.0 v1:172.21.15.92:6805/7087154 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f471c05b9e0 con 0x7f471c057d30 2024-09-21T09:43:32.305 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.304+0000 7f473575f640 1 -- 172.21.15.92:0/4003762884 --> v1:172.21.15.92:6805/7087154 -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) -- 0x7f47301c4e60 con 0x7f471c057d30 2024-09-21T09:43:32.305 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.304+0000 7f47177fe640 1 -- 172.21.15.92:0/4003762884 <== osd.0 v1:172.21.15.92:6805/7087154 2 ==== command_reply(tid 2: 0 ) ==== 8+0+565 (unknown 0 0 125856537) 0x7f47301c4e60 con 0x7f471c057d30 2024-09-21T09:43:32.306 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2024-09-21T09:43:32.306 INFO:teuthology.orchestra.run.smithi092.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2024-09-21T09:43:32.306 INFO:teuthology.orchestra.run.smithi092.stdout:} 2024-09-21T09:43:32.307 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.304+0000 7f473575f640 1 -- 172.21.15.92:0/4003762884 >> v1:172.21.15.92:6805/7087154 conn(0x7f471c057d30 legacy=0x7f471c05a170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:32.307 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.304+0000 7f473575f640 1 -- 172.21.15.92:0/4003762884 >> v1:172.21.15.92:6800/2299163855 conn(0x7f471c0514b0 legacy=0x7f471c053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:32.307 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.304+0000 7f473575f640 1 -- 172.21.15.92:0/4003762884 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f47300957e0 msgr2=0x7f47301266a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:32.307 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.304+0000 7f473575f640 1 --2- 172.21.15.92:0/4003762884 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f47300957e0 0x7f47301266a0 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f4720009780 tx=0x7f4720032b90 comp rx=0 tx=0).stop 2024-09-21T09:43:32.307 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.304+0000 7f472f7fe640 1 -- 172.21.15.92:0/4003762884 reap_dead start 2024-09-21T09:43:32.307 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.304+0000 7f473575f640 1 -- 172.21.15.92:0/4003762884 shutdown_connections 2024-09-21T09:43:32.307 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.304+0000 7f473575f640 1 -- 172.21.15.92:0/4003762884 >> 172.21.15.92:0/4003762884 conn(0x7f473009a3f0 msgr2=0x7f473008bf80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:32.307 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.304+0000 7f473575f640 1 -- 172.21.15.92:0/4003762884 shutdown_connections 2024-09-21T09:43:32.307 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.304+0000 7f473575f640 1 -- 172.21.15.92:0/4003762884 wait complete. 2024-09-21T09:43:32.317 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 0 2024-09-21T09:43:32.420 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.418+0000 7f11786cb640 1 -- 172.21.15.92:0/1333388167 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f11700715f0 msgr2=0x7f1170071a50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:32.420 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.418+0000 7f11786cb640 1 --2- 172.21.15.92:0/1333388167 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f11700715f0 0x7f1170071a50 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f1160009830 tx=0x7f116002a360 comp rx=0 tx=0).stop 2024-09-21T09:43:32.420 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.418+0000 7f11786cb640 1 -- 172.21.15.92:0/1333388167 shutdown_connections 2024-09-21T09:43:32.420 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.418+0000 7f11786cb640 1 --2- 172.21.15.92:0/1333388167 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f1170071f90 0x7f1170133ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.420 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.418+0000 7f11786cb640 1 --2- 172.21.15.92:0/1333388167 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f11700715f0 0x7f1170071a50 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.421 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.418+0000 7f11786cb640 1 --2- 172.21.15.92:0/1333388167 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1170122e80 0x7f1170123260 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.421 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.418+0000 7f11786cb640 1 -- 172.21.15.92:0/1333388167 >> 172.21.15.92:0/1333388167 conn(0x7f117009d3a0 msgr2=0x7f117009d7b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:32.421 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.418+0000 7f11786cb640 1 -- 172.21.15.92:0/1333388167 shutdown_connections 2024-09-21T09:43:32.421 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.418+0000 7f11786cb640 1 -- 172.21.15.92:0/1333388167 wait complete. 2024-09-21T09:43:32.423 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.419+0000 7f11786cb640 1 Processor -- start 2024-09-21T09:43:32.423 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.419+0000 7f11786cb640 1 -- start start 2024-09-21T09:43:32.423 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.419+0000 7f11786cb640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f11700715f0 0x7f1170122180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:32.423 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.419+0000 7f11786cb640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1170071f90 0x7f11701226c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:32.423 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.419+0000 7f11786cb640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1170122e80 0x7f117006f3e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:32.423 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.419+0000 7f11786cb640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f11701344a0 con 0x7f1170071f90 2024-09-21T09:43:32.423 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.419+0000 7f11786cb640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f1170134320 con 0x7f11700715f0 2024-09-21T09:43:32.423 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.419+0000 7f11786cb640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f1170134620 con 0x7f1170122e80 2024-09-21T09:43:32.423 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.419+0000 7f1176467640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f11700715f0 0x7f1170122180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:32.423 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.419+0000 7f1175c66640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1170071f90 0x7f11701226c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:32.423 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.419+0000 7f1176c68640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1170122e80 0x7f117006f3e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:32.424 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.420+0000 7f1175c66640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1170071f90 0x7f11701226c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:43092/0 (socket says 172.21.15.92:43092) 2024-09-21T09:43:32.424 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.420+0000 7f1176c68640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1170122e80 0x7f117006f3e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:48426/0 (socket says 172.21.15.92:48426) 2024-09-21T09:43:32.424 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.420+0000 7f1175c66640 1 -- 172.21.15.92:0/3729388548 learned_addr learned my addr 172.21.15.92:0/3729388548 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:32.424 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.420+0000 7f1176c68640 1 -- 172.21.15.92:0/3729388548 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f11700715f0 msgr2=0x7f1170122180 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:32.424 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.420+0000 7f1176c68640 1 --2- 172.21.15.92:0/3729388548 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f11700715f0 0x7f1170122180 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.424 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.420+0000 7f1176c68640 1 -- 172.21.15.92:0/3729388548 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1170071f90 msgr2=0x7f11701226c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:32.424 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.420+0000 7f1176c68640 1 --2- 172.21.15.92:0/3729388548 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1170071f90 0x7f11701226c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.424 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.420+0000 7f1176c68640 1 -- 172.21.15.92:0/3729388548 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f117012fad0 con 0x7f1170122e80 2024-09-21T09:43:32.424 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.420+0000 7f1175c66640 1 --2- 172.21.15.92:0/3729388548 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1170071f90 0x7f11701226c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:32.424 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.420+0000 7f1176c68640 1 --2- 172.21.15.92:0/3729388548 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1170122e80 0x7f117006f3e0 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f116800efc0 tx=0x7f116800c800 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:32.424 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.420+0000 7f1166ffd640 1 -- 172.21.15.92:0/3729388548 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1168003970 con 0x7f1170122e80 2024-09-21T09:43:32.424 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.420+0000 7f1166ffd640 1 -- 172.21.15.92:0/3729388548 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f1168003c00 con 0x7f1170122e80 2024-09-21T09:43:32.424 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.420+0000 7f11786cb640 1 -- 172.21.15.92:0/3729388548 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1170133420 con 0x7f1170122e80 2024-09-21T09:43:32.425 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.420+0000 7f1166ffd640 1 -- 172.21.15.92:0/3729388548 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1168004020 con 0x7f1170122e80 2024-09-21T09:43:32.425 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.420+0000 7f11786cb640 1 -- 172.21.15.92:0/3729388548 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f117006f920 con 0x7f1170122e80 2024-09-21T09:43:32.425 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.421+0000 7f11786cb640 1 -- 172.21.15.92:0/3729388548 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1170071a50 con 0x7f1170122e80 2024-09-21T09:43:32.425 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.421+0000 7f1166ffd640 1 -- 172.21.15.92:0/3729388548 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f116801d020 con 0x7f1170122e80 2024-09-21T09:43:32.425 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.422+0000 7f1166ffd640 1 -- 172.21.15.92:0/3729388548 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f1168069590 con 0x7f1170122e80 2024-09-21T09:43:32.427 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.425+0000 7f1166ffd640 1 -- 172.21.15.92:0/3729388548 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f117012fad0 con 0x7f1170122e80 2024-09-21T09:43:32.576 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.574+0000 7f11786cb640 1 -- 172.21.15.92:0/3729388548 --> v1:172.21.15.92:6800/2299163855 -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "0", "target": ["mon-mgr", ""]}) -- 0x7f11700699c0 con 0x7f11540514b0 2024-09-21T09:43:32.576 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.574+0000 7f1166ffd640 1 -- 172.21.15.92:0/3729388548 <== mgr.14102 v1:172.21.15.92:6800/2299163855 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 0 to deep-scrub) ==== 41+0+0 (unknown 2057319476 0 0) 0x7f11700699c0 con 0x7f11540514b0 2024-09-21T09:43:32.576 INFO:teuthology.orchestra.run.smithi092.stderr:instructed osd(s) 0 to deep-scrub 2024-09-21T09:43:32.578 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.576+0000 7f11786cb640 1 -- 172.21.15.92:0/3729388548 >> v1:172.21.15.92:6800/2299163855 conn(0x7f11540514b0 legacy=0x7f1154053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:32.578 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.576+0000 7f11786cb640 1 -- 172.21.15.92:0/3729388548 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1170122e80 msgr2=0x7f117006f3e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:32.578 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.576+0000 7f11786cb640 1 --2- 172.21.15.92:0/3729388548 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1170122e80 0x7f117006f3e0 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f116800efc0 tx=0x7f116800c800 comp rx=0 tx=0).stop 2024-09-21T09:43:32.578 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.576+0000 7f11786cb640 1 -- 172.21.15.92:0/3729388548 shutdown_connections 2024-09-21T09:43:32.578 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.576+0000 7f11786cb640 1 --2- 172.21.15.92:0/3729388548 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f1170122e80 0x7f117006f3e0 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.578 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.576+0000 7f11786cb640 1 --2- 172.21.15.92:0/3729388548 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f1170071f90 0x7f11701226c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.578 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.576+0000 7f11786cb640 1 --2- 172.21.15.92:0/3729388548 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f11700715f0 0x7f1170122180 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.578 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.576+0000 7f11786cb640 1 -- 172.21.15.92:0/3729388548 >> 172.21.15.92:0/3729388548 conn(0x7f117009d3a0 msgr2=0x7f1170125b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:32.579 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.576+0000 7f11786cb640 1 -- 172.21.15.92:0/3729388548 shutdown_connections 2024-09-21T09:43:32.579 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.576+0000 7f11786cb640 1 -- 172.21.15.92:0/3729388548 wait complete. 2024-09-21T09:43:32.588 INFO:tasks.ceph:Scrubbing osd.1 2024-09-21T09:43:32.588 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 config set osd_debug_deep_scrub_sleep 0 2024-09-21T09:43:32.692 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.690+0000 7faddedf6640 1 -- 172.21.15.92:0/3674437012 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fadd80957e0 msgr2=0x7fadd8095c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:32.692 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.690+0000 7faddedf6640 1 --2- 172.21.15.92:0/3674437012 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fadd80957e0 0x7fadd8095c20 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7fadc8009830 tx=0x7fadc802a3e0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.693 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.690+0000 7faddedf6640 1 -- 172.21.15.92:0/3674437012 shutdown_connections 2024-09-21T09:43:32.693 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.690+0000 7faddedf6640 1 --2- 172.21.15.92:0/3674437012 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fadd80957e0 0x7fadd8095c20 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.693 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.690+0000 7faddedf6640 1 --2- 172.21.15.92:0/3674437012 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fadd812bc10 0x7fadd812bff0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.693 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.690+0000 7faddedf6640 1 -- 172.21.15.92:0/3674437012 >> 172.21.15.92:0/3674437012 conn(0x7fadd809a3f0 msgr2=0x7fadd8097bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:32.693 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.690+0000 7faddedf6640 1 -- 172.21.15.92:0/3674437012 shutdown_connections 2024-09-21T09:43:32.693 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.690+0000 7faddedf6640 1 -- 172.21.15.92:0/3674437012 wait complete. 2024-09-21T09:43:32.695 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.691+0000 7faddedf6640 1 Processor -- start 2024-09-21T09:43:32.695 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.691+0000 7faddedf6640 1 -- start start 2024-09-21T09:43:32.695 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.692+0000 7faddedf6640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fadd80957e0 0x7fadd808e190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:32.695 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.692+0000 7faddedf6640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fadd8096160 0x7fadd808e6d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:32.695 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.692+0000 7faddedf6640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fadd812bc10 0x7fadd808ec10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:32.695 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.692+0000 7faddedf6640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fadd81347c0 con 0x7fadd812bc10 2024-09-21T09:43:32.696 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.692+0000 7faddedf6640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fadd8134640 con 0x7fadd80957e0 2024-09-21T09:43:32.696 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.692+0000 7faddedf6640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fadd8134940 con 0x7fadd8096160 2024-09-21T09:43:32.696 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.692+0000 7faddcb92640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fadd80957e0 0x7fadd808e190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:32.696 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.692+0000 7faddcb92640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fadd80957e0 0x7fadd808e190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:44696/0 (socket says 172.21.15.92:44696) 2024-09-21T09:43:32.696 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.692+0000 7faddcb92640 1 -- 172.21.15.92:0/3437247969 learned_addr learned my addr 172.21.15.92:0/3437247969 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:32.696 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.692+0000 7fadcffff640 1 --2- 172.21.15.92:0/3437247969 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fadd8096160 0x7fadd808e6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:32.696 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.692+0000 7faddd393640 1 --2- 172.21.15.92:0/3437247969 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fadd812bc10 0x7fadd808ec10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:32.696 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.692+0000 7faddcb92640 1 -- 172.21.15.92:0/3437247969 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fadd8096160 msgr2=0x7fadd808e6d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:32.696 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.692+0000 7faddcb92640 1 --2- 172.21.15.92:0/3437247969 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fadd8096160 0x7fadd808e6d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.696 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.692+0000 7faddcb92640 1 -- 172.21.15.92:0/3437247969 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fadd812bc10 msgr2=0x7fadd808ec10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:32.696 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.692+0000 7faddcb92640 1 --2- 172.21.15.92:0/3437247969 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fadd812bc10 0x7fadd808ec10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.696 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.692+0000 7faddcb92640 1 -- 172.21.15.92:0/3437247969 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fadd8094940 con 0x7fadd80957e0 2024-09-21T09:43:32.696 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.692+0000 7faddcb92640 1 --2- 172.21.15.92:0/3437247969 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fadd80957e0 0x7fadd808e190 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7fadd400e9c0 tx=0x7fadd400ee80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:32.697 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.692+0000 7fadcd7fa640 1 -- 172.21.15.92:0/3437247969 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fadd400cde0 con 0x7fadd80957e0 2024-09-21T09:43:32.697 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.692+0000 7fadcd7fa640 1 -- 172.21.15.92:0/3437247969 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fadd4004020 con 0x7fadd80957e0 2024-09-21T09:43:32.697 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.692+0000 7faddedf6640 1 -- 172.21.15.92:0/3437247969 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fadd8093fa0 con 0x7fadd80957e0 2024-09-21T09:43:32.697 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.692+0000 7fadcd7fa640 1 -- 172.21.15.92:0/3437247969 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fadd4004450 con 0x7fadd80957e0 2024-09-21T09:43:32.697 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.692+0000 7faddedf6640 1 -- 172.21.15.92:0/3437247969 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fadd808f6c0 con 0x7fadd80957e0 2024-09-21T09:43:32.697 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.693+0000 7faddedf6640 1 -- 172.21.15.92:0/3437247969 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_get_version(what=osdmap handle=1) -- 0x7fadd8095c20 con 0x7fadd80957e0 2024-09-21T09:43:32.697 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.693+0000 7fadcd7fa640 1 -- 172.21.15.92:0/3437247969 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7fadd40035f0 con 0x7fadd80957e0 2024-09-21T09:43:32.698 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.694+0000 7fadcd7fa640 1 -- 172.21.15.92:0/3437247969 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fadd4069940 con 0x7fadd80957e0 2024-09-21T09:43:32.698 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.694+0000 7fadcd7fa640 1 -- 172.21.15.92:0/3437247969 --> v1:172.21.15.92:6801/2577383826 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7fadc405b9e0 con 0x7fadc4057d30 2024-09-21T09:43:32.698 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.694+0000 7fadcd7fa640 1 -- 172.21.15.92:0/3437247969 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7fadd406e2c0 con 0x7fadd80957e0 2024-09-21T09:43:32.698 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.695+0000 7fadcd7fa640 1 -- 172.21.15.92:0/3437247969 <== osd.1 v1:172.21.15.92:6801/2577383826 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7fadc405b9e0 con 0x7fadc4057d30 2024-09-21T09:43:32.708 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.707+0000 7faddedf6640 1 -- 172.21.15.92:0/3437247969 --> v1:172.21.15.92:6801/2577383826 -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) -- 0x7fadd812aa60 con 0x7fadc4057d30 2024-09-21T09:43:32.709 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.707+0000 7fadcd7fa640 1 -- 172.21.15.92:0/3437247969 <== osd.1 v1:172.21.15.92:6801/2577383826 2 ==== command_reply(tid 2: 0 ) ==== 8+0+565 (unknown 0 0 125856537) 0x7fadd812aa60 con 0x7fadc4057d30 2024-09-21T09:43:32.709 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2024-09-21T09:43:32.709 INFO:teuthology.orchestra.run.smithi092.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2024-09-21T09:43:32.709 INFO:teuthology.orchestra.run.smithi092.stdout:} 2024-09-21T09:43:32.710 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.707+0000 7faddedf6640 1 -- 172.21.15.92:0/3437247969 >> v1:172.21.15.92:6801/2577383826 conn(0x7fadc4057d30 legacy=0x7fadc405a170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:32.710 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.708+0000 7faddedf6640 1 -- 172.21.15.92:0/3437247969 >> v1:172.21.15.92:6800/2299163855 conn(0x7fadc40514b0 legacy=0x7fadc4053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:32.710 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.708+0000 7faddedf6640 1 -- 172.21.15.92:0/3437247969 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fadd80957e0 msgr2=0x7fadd808e190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:32.710 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.708+0000 7faddedf6640 1 --2- 172.21.15.92:0/3437247969 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fadd80957e0 0x7fadd808e190 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7fadd400e9c0 tx=0x7fadd400ee80 comp rx=0 tx=0).stop 2024-09-21T09:43:32.710 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.708+0000 7faddd393640 1 -- 172.21.15.92:0/3437247969 reap_dead start 2024-09-21T09:43:32.711 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.708+0000 7faddedf6640 1 -- 172.21.15.92:0/3437247969 shutdown_connections 2024-09-21T09:43:32.711 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.708+0000 7faddedf6640 1 -- 172.21.15.92:0/3437247969 >> 172.21.15.92:0/3437247969 conn(0x7fadd809a3f0 msgr2=0x7fadd812a100 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:32.711 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.708+0000 7faddedf6640 1 -- 172.21.15.92:0/3437247969 shutdown_connections 2024-09-21T09:43:32.711 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.708+0000 7faddedf6640 1 -- 172.21.15.92:0/3437247969 wait complete. 2024-09-21T09:43:32.720 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1 2024-09-21T09:43:32.821 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.819+0000 7f61c56d4640 1 -- 172.21.15.92:0/3697483218 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61c012b490 msgr2=0x7f61c012b870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:32.821 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.819+0000 7f61c56d4640 1 --2- 172.21.15.92:0/3697483218 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61c012b490 0x7f61c012b870 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7f61b0009860 tx=0x7f61b002a4b0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.821 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.819+0000 7f61c56d4640 1 -- 172.21.15.92:0/3697483218 shutdown_connections 2024-09-21T09:43:32.821 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.819+0000 7f61c56d4640 1 --2- 172.21.15.92:0/3697483218 >> v2:172.21.15.92:3300/0 conn(0x7f61c012bdb0 0x7f61c0094e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.822 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.819+0000 7f61c56d4640 1 --2- 172.21.15.92:0/3697483218 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61c012b490 0x7f61c012b870 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.822 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.819+0000 7f61c56d4640 1 -- 172.21.15.92:0/3697483218 >> 172.21.15.92:0/3697483218 conn(0x7f61c009afd0 msgr2=0x7f61c009b3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:32.822 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.819+0000 7f61c56d4640 1 -- 172.21.15.92:0/3697483218 shutdown_connections 2024-09-21T09:43:32.822 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.819+0000 7f61c56d4640 1 -- 172.21.15.92:0/3697483218 wait complete. 2024-09-21T09:43:32.824 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.820+0000 7f61c56d4640 1 Processor -- start 2024-09-21T09:43:32.824 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.820+0000 7f61c56d4640 1 -- start start 2024-09-21T09:43:32.824 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.820+0000 7f61c56d4640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61c00953a0 0x7f61c01c0bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:32.824 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61c56d4640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61c012b490 0x7f61c01c1100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:32.824 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61c56d4640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f61c012bdb0 0x7f61c012f170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:32.824 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61c56d4640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f61c0133600 con 0x7f61c00953a0 2024-09-21T09:43:32.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61c56d4640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f61c0133480 con 0x7f61c012bdb0 2024-09-21T09:43:32.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61c56d4640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f61c0133780 con 0x7f61c012b490 2024-09-21T09:43:32.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61beffd640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61c00953a0 0x7f61c01c0bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:32.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61be7fc640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61c012b490 0x7f61c01c1100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:32.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61beffd640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61c00953a0 0x7f61c01c0bc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:43124/0 (socket says 172.21.15.92:43124) 2024-09-21T09:43:32.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61beffd640 1 -- 172.21.15.92:0/2009958241 learned_addr learned my addr 172.21.15.92:0/2009958241 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:32.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61be7fc640 1 --2- 172.21.15.92:0/2009958241 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61c012b490 0x7f61c01c1100 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3302/0 says I am v2:172.21.15.92:48454/0 (socket says 172.21.15.92:48454) 2024-09-21T09:43:32.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61bf7fe640 1 --2- 172.21.15.92:0/2009958241 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f61c012bdb0 0x7f61c012f170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:32.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61beffd640 1 -- 172.21.15.92:0/2009958241 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61c012b490 msgr2=0x7f61c01c1100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:32.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61beffd640 1 --2- 172.21.15.92:0/2009958241 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61c012b490 0x7f61c01c1100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61beffd640 1 -- 172.21.15.92:0/2009958241 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f61c012bdb0 msgr2=0x7f61c012f170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:32.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61beffd640 1 --2- 172.21.15.92:0/2009958241 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f61c012bdb0 0x7f61c012f170 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.825 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61beffd640 1 -- 172.21.15.92:0/2009958241 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f61c0093850 con 0x7f61c00953a0 2024-09-21T09:43:32.826 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61be7fc640 1 --2- 172.21.15.92:0/2009958241 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61c012b490 0x7f61c01c1100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:32.826 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61bf7fe640 1 --2- 172.21.15.92:0/2009958241 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f61c012bdb0 0x7f61c012f170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:32.826 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61beffd640 1 --2- 172.21.15.92:0/2009958241 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61c00953a0 0x7f61c01c0bc0 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f61b0000c00 tx=0x7f61b0032c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:32.826 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61a77fe640 1 -- 172.21.15.92:0/2009958241 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f61b0040070 con 0x7f61c00953a0 2024-09-21T09:43:32.826 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61a77fe640 1 -- 172.21.15.92:0/2009958241 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f61b00038e0 con 0x7f61c00953a0 2024-09-21T09:43:32.826 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61a77fe640 1 -- 172.21.15.92:0/2009958241 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f61b0003d30 con 0x7f61c00953a0 2024-09-21T09:43:32.826 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61c56d4640 1 -- 172.21.15.92:0/2009958241 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f61c006e860 con 0x7f61c00953a0 2024-09-21T09:43:32.826 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.821+0000 7f61c56d4640 1 -- 172.21.15.92:0/2009958241 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f61c01c1640 con 0x7f61c00953a0 2024-09-21T09:43:32.826 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.822+0000 7f61c56d4640 1 -- 172.21.15.92:0/2009958241 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f61c0131a70 con 0x7f61c00953a0 2024-09-21T09:43:32.826 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.823+0000 7f61a77fe640 1 -- 172.21.15.92:0/2009958241 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f61b0039040 con 0x7f61c00953a0 2024-09-21T09:43:32.826 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.823+0000 7f61a77fe640 1 -- 172.21.15.92:0/2009958241 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f61b008a640 con 0x7f61c00953a0 2024-09-21T09:43:32.828 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.826+0000 7f61a77fe640 1 -- 172.21.15.92:0/2009958241 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f61b00896f0 con 0x7f61c00953a0 2024-09-21T09:43:32.976 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.974+0000 7f61c56d4640 1 -- 172.21.15.92:0/2009958241 --> v1:172.21.15.92:6800/2299163855 -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "1", "target": ["mon-mgr", ""]}) -- 0x7f61c012b870 con 0x7f61ac0514b0 2024-09-21T09:43:32.976 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.975+0000 7f61a77fe640 1 -- 172.21.15.92:0/2009958241 <== mgr.14102 v1:172.21.15.92:6800/2299163855 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 1 to deep-scrub) ==== 41+0+0 (unknown 489605429 0 0) 0x7f61c012b870 con 0x7f61ac0514b0 2024-09-21T09:43:32.977 INFO:teuthology.orchestra.run.smithi092.stderr:instructed osd(s) 1 to deep-scrub 2024-09-21T09:43:32.979 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.977+0000 7f61c56d4640 1 -- 172.21.15.92:0/2009958241 >> v1:172.21.15.92:6800/2299163855 conn(0x7f61ac0514b0 legacy=0x7f61ac053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:32.979 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.977+0000 7f61c56d4640 1 -- 172.21.15.92:0/2009958241 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61c00953a0 msgr2=0x7f61c01c0bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:32.979 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.977+0000 7f61c56d4640 1 --2- 172.21.15.92:0/2009958241 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61c00953a0 0x7f61c01c0bc0 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f61b0000c00 tx=0x7f61b0032c10 comp rx=0 tx=0).stop 2024-09-21T09:43:32.979 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.977+0000 7f61c56d4640 1 -- 172.21.15.92:0/2009958241 shutdown_connections 2024-09-21T09:43:32.979 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.977+0000 7f61c56d4640 1 --2- 172.21.15.92:0/2009958241 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f61c012bdb0 0x7f61c012f170 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.979 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.977+0000 7f61c56d4640 1 --2- 172.21.15.92:0/2009958241 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f61c012b490 0x7f61c01c1100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.979 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.977+0000 7f61c56d4640 1 --2- 172.21.15.92:0/2009958241 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f61c00953a0 0x7f61c01c0bc0 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:32.979 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.977+0000 7f61c56d4640 1 -- 172.21.15.92:0/2009958241 >> 172.21.15.92:0/2009958241 conn(0x7f61c009afd0 msgr2=0x7f61c0092e00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:32.979 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.977+0000 7f61c56d4640 1 -- 172.21.15.92:0/2009958241 shutdown_connections 2024-09-21T09:43:32.979 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:32.977+0000 7f61c56d4640 1 -- 172.21.15.92:0/2009958241 wait complete. 2024-09-21T09:43:32.989 INFO:tasks.ceph:Scrubbing osd.2 2024-09-21T09:43:32.989 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 config set osd_debug_deep_scrub_sleep 0 2024-09-21T09:43:33.093 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.091+0000 7ff4fcf83640 1 -- 172.21.15.92:0/53799749 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff4f8073b60 msgr2=0x7ff4f8073f40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.093 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.091+0000 7ff4fcf83640 1 --2- 172.21.15.92:0/53799749 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff4f8073b60 0x7ff4f8073f40 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7ff4e8009900 tx=0x7ff4e802a440 comp rx=0 tx=0).stop 2024-09-21T09:43:33.093 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.091+0000 7ff4fcf83640 1 -- 172.21.15.92:0/53799749 shutdown_connections 2024-09-21T09:43:33.093 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.091+0000 7ff4fcf83640 1 --2- 172.21.15.92:0/53799749 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff4f8096a30 0x7ff4f8096ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.093 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.091+0000 7ff4fcf83640 1 --2- 172.21.15.92:0/53799749 >> v2:172.21.15.92:3300/0 conn(0x7ff4f809e8f0 0x7ff4f80964f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.093 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.091+0000 7ff4fcf83640 1 --2- 172.21.15.92:0/53799749 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff4f8073b60 0x7ff4f8073f40 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.093 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.091+0000 7ff4fcf83640 1 -- 172.21.15.92:0/53799749 >> 172.21.15.92:0/53799749 conn(0x7ff4f80a1aa0 msgr2=0x7ff4f809df70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:33.093 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.091+0000 7ff4fcf83640 1 -- 172.21.15.92:0/53799749 shutdown_connections 2024-09-21T09:43:33.093 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.091+0000 7ff4fcf83640 1 -- 172.21.15.92:0/53799749 wait complete. 2024-09-21T09:43:33.095 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.092+0000 7ff4fcf83640 1 Processor -- start 2024-09-21T09:43:33.095 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.092+0000 7ff4fcf83640 1 -- start start 2024-09-21T09:43:33.095 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.092+0000 7ff4fcf83640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff4f8073b60 0x7ff4f8090390 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:33.096 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.092+0000 7ff4fcf83640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff4f8096a30 0x7ff4f80908d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:33.096 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.092+0000 7ff4fcf83640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff4f809e8f0 0x7ff4f8090e10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:33.096 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.092+0000 7ff4fcf83640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7ff4f8136960 con 0x7ff4f8096a30 2024-09-21T09:43:33.096 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.092+0000 7ff4fcf83640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7ff4f81367e0 con 0x7ff4f8073b60 2024-09-21T09:43:33.096 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.092+0000 7ff4fcf83640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7ff4f8136ae0 con 0x7ff4f809e8f0 2024-09-21T09:43:33.096 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.092+0000 7ff4f659c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff4f8073b60 0x7ff4f8090390 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:33.096 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.092+0000 7ff4f659c640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff4f8073b60 0x7ff4f8090390 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:44722/0 (socket says 172.21.15.92:44722) 2024-09-21T09:43:33.096 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.092+0000 7ff4f659c640 1 -- 172.21.15.92:0/4030201931 learned_addr learned my addr 172.21.15.92:0/4030201931 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:33.096 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.092+0000 7ff4effff640 1 --2- 172.21.15.92:0/4030201931 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff4f8096a30 0x7ff4f80908d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:33.096 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.092+0000 7ff4f6d9d640 1 --2- 172.21.15.92:0/4030201931 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff4f809e8f0 0x7ff4f8090e10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:33.096 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.093+0000 7ff4effff640 1 -- 172.21.15.92:0/4030201931 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff4f809e8f0 msgr2=0x7ff4f8090e10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.096 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.093+0000 7ff4effff640 1 --2- 172.21.15.92:0/4030201931 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff4f809e8f0 0x7ff4f8090e10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.096 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.093+0000 7ff4effff640 1 -- 172.21.15.92:0/4030201931 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff4f8073b60 msgr2=0x7ff4f8090390 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.096 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.093+0000 7ff4effff640 1 --2- 172.21.15.92:0/4030201931 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff4f8073b60 0x7ff4f8090390 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.096 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.093+0000 7ff4effff640 1 -- 172.21.15.92:0/4030201931 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff4f809ace0 con 0x7ff4f8096a30 2024-09-21T09:43:33.097 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.093+0000 7ff4f6d9d640 1 --2- 172.21.15.92:0/4030201931 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff4f809e8f0 0x7ff4f8090e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:33.097 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.093+0000 7ff4f659c640 1 --2- 172.21.15.92:0/4030201931 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff4f8073b60 0x7ff4f8090390 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:33.097 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.093+0000 7ff4effff640 1 --2- 172.21.15.92:0/4030201931 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff4f8096a30 0x7ff4f80908d0 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7ff4e000e9f0 tx=0x7ff4e000edf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:33.097 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.093+0000 7ff4eeffd640 1 -- 172.21.15.92:0/4030201931 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff4e000cc60 con 0x7ff4f8096a30 2024-09-21T09:43:33.097 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.093+0000 7ff4eeffd640 1 -- 172.21.15.92:0/4030201931 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7ff4e000ce00 con 0x7ff4f8096a30 2024-09-21T09:43:33.097 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.093+0000 7ff4eeffd640 1 -- 172.21.15.92:0/4030201931 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff4e0004280 con 0x7ff4f8096a30 2024-09-21T09:43:33.097 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.093+0000 7ff4fcf83640 1 -- 172.21.15.92:0/4030201931 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff4f8129560 con 0x7ff4f8096a30 2024-09-21T09:43:33.097 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.093+0000 7ff4fcf83640 1 -- 172.21.15.92:0/4030201931 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff4f8091e60 con 0x7ff4f8096a30 2024-09-21T09:43:33.097 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.094+0000 7ff4fcf83640 1 -- 172.21.15.92:0/4030201931 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7ff4f8096ec0 con 0x7ff4f8096a30 2024-09-21T09:43:33.098 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.094+0000 7ff4eeffd640 1 -- 172.21.15.92:0/4030201931 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7ff4e00035f0 con 0x7ff4f8096a30 2024-09-21T09:43:33.098 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.094+0000 7ff4eeffd640 1 -- 172.21.15.92:0/4030201931 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7ff4e0069770 con 0x7ff4f8096a30 2024-09-21T09:43:33.098 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.095+0000 7ff4eeffd640 1 -- 172.21.15.92:0/4030201931 --> v1:172.21.15.92:6809/4235634095 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7ff4d805ba10 con 0x7ff4d8057d60 2024-09-21T09:43:33.098 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.095+0000 7ff4eeffd640 1 -- 172.21.15.92:0/4030201931 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7ff4e0070050 con 0x7ff4f8096a30 2024-09-21T09:43:33.098 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.095+0000 7ff4eeffd640 1 -- 172.21.15.92:0/4030201931 <== osd.2 v1:172.21.15.92:6809/4235634095 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7ff4d805ba10 con 0x7ff4d8057d60 2024-09-21T09:43:33.110 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.108+0000 7ff4fcf83640 1 -- 172.21.15.92:0/4030201931 --> v1:172.21.15.92:6809/4235634095 -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) -- 0x7ff4f8128790 con 0x7ff4d8057d60 2024-09-21T09:43:33.110 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.108+0000 7ff4eeffd640 1 -- 172.21.15.92:0/4030201931 <== osd.2 v1:172.21.15.92:6809/4235634095 2 ==== command_reply(tid 2: 0 ) ==== 8+0+565 (unknown 0 0 125856537) 0x7ff4f8128790 con 0x7ff4d8057d60 2024-09-21T09:43:33.110 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2024-09-21T09:43:33.111 INFO:teuthology.orchestra.run.smithi092.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2024-09-21T09:43:33.111 INFO:teuthology.orchestra.run.smithi092.stdout:} 2024-09-21T09:43:33.112 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.108+0000 7ff4fcf83640 1 -- 172.21.15.92:0/4030201931 >> v1:172.21.15.92:6809/4235634095 conn(0x7ff4d8057d60 legacy=0x7ff4d805a1a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.112 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.108+0000 7ff4fcf83640 1 -- 172.21.15.92:0/4030201931 >> v1:172.21.15.92:6800/2299163855 conn(0x7ff4d80514e0 legacy=0x7ff4d80539a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.112 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.108+0000 7ff4fcf83640 1 -- 172.21.15.92:0/4030201931 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff4f8096a30 msgr2=0x7ff4f80908d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.112 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.108+0000 7ff4fcf83640 1 --2- 172.21.15.92:0/4030201931 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff4f8096a30 0x7ff4f80908d0 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7ff4e000e9f0 tx=0x7ff4e000edf0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.112 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.108+0000 7ff4f6d9d640 1 -- 172.21.15.92:0/4030201931 reap_dead start 2024-09-21T09:43:33.113 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.108+0000 7ff4fcf83640 1 -- 172.21.15.92:0/4030201931 shutdown_connections 2024-09-21T09:43:33.113 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.108+0000 7ff4fcf83640 1 -- 172.21.15.92:0/4030201931 >> 172.21.15.92:0/4030201931 conn(0x7ff4f80a1aa0 msgr2=0x7ff4f812b9d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:33.113 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.108+0000 7ff4fcf83640 1 -- 172.21.15.92:0/4030201931 shutdown_connections 2024-09-21T09:43:33.113 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.108+0000 7ff4fcf83640 1 -- 172.21.15.92:0/4030201931 wait complete. 2024-09-21T09:43:33.121 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 2 2024-09-21T09:43:33.220 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.218+0000 7f4a24d62640 1 -- 172.21.15.92:0/4216267454 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4a2012c3f0 msgr2=0x7f4a20128260 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.220 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.218+0000 7f4a24d62640 1 --2- 172.21.15.92:0/4216267454 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4a2012c3f0 0x7f4a20128260 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f4a14005fd0 tx=0x7f4a1402a540 comp rx=0 tx=0).stop 2024-09-21T09:43:33.220 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.218+0000 7f4a24d62640 1 -- 172.21.15.92:0/4216267454 shutdown_connections 2024-09-21T09:43:33.220 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.218+0000 7f4a24d62640 1 --2- 172.21.15.92:0/4216267454 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4a2012c3f0 0x7f4a20128260 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.220 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.218+0000 7f4a24d62640 1 --2- 172.21.15.92:0/4216267454 >> v2:172.21.15.92:3300/0 conn(0x7f4a2012bad0 0x7f4a2012beb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.221 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.218+0000 7f4a24d62640 1 -- 172.21.15.92:0/4216267454 >> 172.21.15.92:0/4216267454 conn(0x7f4a2009afd0 msgr2=0x7f4a2009b3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:33.221 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.218+0000 7f4a24d62640 1 -- 172.21.15.92:0/4216267454 shutdown_connections 2024-09-21T09:43:33.221 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.218+0000 7f4a24d62640 1 -- 172.21.15.92:0/4216267454 wait complete. 2024-09-21T09:43:33.224 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.219+0000 7f4a24d62640 1 Processor -- start 2024-09-21T09:43:33.224 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.219+0000 7f4a24d62640 1 -- start start 2024-09-21T09:43:33.224 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.219+0000 7f4a24d62640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4a201287a0 0x7f4a2012f3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:33.224 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.219+0000 7f4a24d62640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4a2012bad0 0x7f4a2012f8e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:33.224 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.219+0000 7f4a24d62640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4a2012c3f0 0x7f4a2012fe20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:33.224 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.219+0000 7f4a24d62640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f4a201346b0 con 0x7f4a201287a0 2024-09-21T09:43:33.224 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.219+0000 7f4a24d62640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f4a20134530 con 0x7f4a2012c3f0 2024-09-21T09:43:33.224 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.219+0000 7f4a24d62640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f4a20134830 con 0x7f4a2012bad0 2024-09-21T09:43:33.224 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.219+0000 7f4a1e59c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4a201287a0 0x7f4a2012f3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:33.224 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.219+0000 7f4a1dd9b640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4a2012bad0 0x7f4a2012f8e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:33.224 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.219+0000 7f4a1e59c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4a201287a0 0x7f4a2012f3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:43158/0 (socket says 172.21.15.92:43158) 2024-09-21T09:43:33.224 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.219+0000 7f4a1e59c640 1 -- 172.21.15.92:0/4284787926 learned_addr learned my addr 172.21.15.92:0/4284787926 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:33.225 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.219+0000 7f4a1ed9d640 1 --2- 172.21.15.92:0/4284787926 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4a2012c3f0 0x7f4a2012fe20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:33.225 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.220+0000 7f4a1e59c640 1 -- 172.21.15.92:0/4284787926 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4a2012bad0 msgr2=0x7f4a2012f8e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.225 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.220+0000 7f4a1e59c640 1 --2- 172.21.15.92:0/4284787926 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4a2012bad0 0x7f4a2012f8e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.225 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.220+0000 7f4a1e59c640 1 -- 172.21.15.92:0/4284787926 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4a2012c3f0 msgr2=0x7f4a2012fe20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.225 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.220+0000 7f4a1e59c640 1 --2- 172.21.15.92:0/4284787926 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4a2012c3f0 0x7f4a2012fe20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.225 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.220+0000 7f4a1e59c640 1 -- 172.21.15.92:0/4284787926 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4a20072f60 con 0x7f4a201287a0 2024-09-21T09:43:33.225 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.220+0000 7f4a1ed9d640 1 --2- 172.21.15.92:0/4284787926 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4a2012c3f0 0x7f4a2012fe20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:33.225 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.220+0000 7f4a1e59c640 1 --2- 172.21.15.92:0/4284787926 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4a201287a0 0x7f4a2012f3a0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f4a1000ea60 tx=0x7f4a1000ef20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:33.225 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.220+0000 7f4a0effd640 1 -- 172.21.15.92:0/4284787926 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a1000cde0 con 0x7f4a201287a0 2024-09-21T09:43:33.225 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.220+0000 7f4a0effd640 1 -- 172.21.15.92:0/4284787926 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f4a10004020 con 0x7f4a201287a0 2024-09-21T09:43:33.225 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.220+0000 7f4a24d62640 1 -- 172.21.15.92:0/4284787926 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4a20095c40 con 0x7f4a201287a0 2024-09-21T09:43:33.225 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.220+0000 7f4a0effd640 1 -- 172.21.15.92:0/4284787926 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a10004450 con 0x7f4a201287a0 2024-09-21T09:43:33.225 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.220+0000 7f4a24d62640 1 -- 172.21.15.92:0/4284787926 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4a200952b0 con 0x7f4a201287a0 2024-09-21T09:43:33.225 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.221+0000 7f4a24d62640 1 -- 172.21.15.92:0/4284787926 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4a2012beb0 con 0x7f4a201287a0 2024-09-21T09:43:33.225 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.221+0000 7f4a0effd640 1 -- 172.21.15.92:0/4284787926 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f4a100035f0 con 0x7f4a201287a0 2024-09-21T09:43:33.226 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.222+0000 7f4a0effd640 1 -- 172.21.15.92:0/4284787926 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f4a10069970 con 0x7f4a201287a0 2024-09-21T09:43:33.227 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.225+0000 7f4a0effd640 1 -- 172.21.15.92:0/4284787926 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f4a20072f60 con 0x7f4a201287a0 2024-09-21T09:43:33.378 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.376+0000 7f4a24d62640 1 -- 172.21.15.92:0/4284787926 --> v1:172.21.15.92:6800/2299163855 -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "2", "target": ["mon-mgr", ""]}) -- 0x7f4a200699c0 con 0x7f4a000514b0 2024-09-21T09:43:33.378 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.376+0000 7f4a0effd640 1 -- 172.21.15.92:0/4284787926 <== mgr.14102 v1:172.21.15.92:6800/2299163855 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 2 to deep-scrub) ==== 41+0+0 (unknown 3049115190 0 0) 0x7f4a200699c0 con 0x7f4a000514b0 2024-09-21T09:43:33.378 INFO:teuthology.orchestra.run.smithi092.stderr:instructed osd(s) 2 to deep-scrub 2024-09-21T09:43:33.380 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.378+0000 7f4a24d62640 1 -- 172.21.15.92:0/4284787926 >> v1:172.21.15.92:6800/2299163855 conn(0x7f4a000514b0 legacy=0x7f4a00053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.380 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.378+0000 7f4a24d62640 1 -- 172.21.15.92:0/4284787926 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4a201287a0 msgr2=0x7f4a2012f3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.380 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.378+0000 7f4a24d62640 1 --2- 172.21.15.92:0/4284787926 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4a201287a0 0x7f4a2012f3a0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f4a1000ea60 tx=0x7f4a1000ef20 comp rx=0 tx=0).stop 2024-09-21T09:43:33.381 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.378+0000 7f4a24d62640 1 -- 172.21.15.92:0/4284787926 shutdown_connections 2024-09-21T09:43:33.381 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.378+0000 7f4a24d62640 1 --2- 172.21.15.92:0/4284787926 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f4a2012c3f0 0x7f4a2012fe20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.381 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.378+0000 7f4a24d62640 1 --2- 172.21.15.92:0/4284787926 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f4a2012bad0 0x7f4a2012f8e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.381 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.378+0000 7f4a24d62640 1 --2- 172.21.15.92:0/4284787926 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f4a201287a0 0x7f4a2012f3a0 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.381 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.378+0000 7f4a24d62640 1 -- 172.21.15.92:0/4284787926 >> 172.21.15.92:0/4284787926 conn(0x7f4a2009afd0 msgr2=0x7f4a20098ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:33.381 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.378+0000 7f4a24d62640 1 -- 172.21.15.92:0/4284787926 shutdown_connections 2024-09-21T09:43:33.381 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.378+0000 7f4a24d62640 1 -- 172.21.15.92:0/4284787926 wait complete. 2024-09-21T09:43:33.391 INFO:tasks.ceph:Scrubbing osd.3 2024-09-21T09:43:33.391 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 config set osd_debug_deep_scrub_sleep 0 2024-09-21T09:43:33.495 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.493+0000 7fbe216b1640 1 -- 172.21.15.92:0/3727675159 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbe1c12beb0 msgr2=0x7fbe1c0946d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.495 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.493+0000 7fbe216b1640 1 --2- 172.21.15.92:0/3727675159 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbe1c12beb0 0x7fbe1c0946d0 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fbe10005fd0 tx=0x7fbe1002a4b0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.495 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.493+0000 7fbe216b1640 1 -- 172.21.15.92:0/3727675159 shutdown_connections 2024-09-21T09:43:33.496 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.493+0000 7fbe216b1640 1 --2- 172.21.15.92:0/3727675159 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbe1c12beb0 0x7fbe1c0946d0 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.496 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.493+0000 7fbe216b1640 1 --2- 172.21.15.92:0/3727675159 >> v2:172.21.15.92:3300/0 conn(0x7fbe1c12b590 0x7fbe1c12b970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.496 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.493+0000 7fbe216b1640 1 -- 172.21.15.92:0/3727675159 >> 172.21.15.92:0/3727675159 conn(0x7fbe1c09a3f0 msgr2=0x7fbe1c097bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:33.496 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.493+0000 7fbe216b1640 1 -- 172.21.15.92:0/3727675159 shutdown_connections 2024-09-21T09:43:33.496 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.493+0000 7fbe216b1640 1 -- 172.21.15.92:0/3727675159 wait complete. 2024-09-21T09:43:33.498 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.494+0000 7fbe216b1640 1 Processor -- start 2024-09-21T09:43:33.498 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.494+0000 7fbe216b1640 1 -- start start 2024-09-21T09:43:33.498 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.495+0000 7fbe216b1640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbe1c096d20 0x7fbe1c08f960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:33.498 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.495+0000 7fbe216b1640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbe1c12b590 0x7fbe1c08fea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:33.498 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.495+0000 7fbe216b1640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbe1c12beb0 0x7fbe1c0903e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:33.498 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.495+0000 7fbe216b1640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fbe1c133700 con 0x7fbe1c12b590 2024-09-21T09:43:33.498 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.495+0000 7fbe216b1640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fbe1c133580 con 0x7fbe1c096d20 2024-09-21T09:43:33.498 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.495+0000 7fbe216b1640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fbe1c133880 con 0x7fbe1c12beb0 2024-09-21T09:43:33.498 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.495+0000 7fbe1affd640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbe1c096d20 0x7fbe1c08f960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:33.498 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.495+0000 7fbe1a7fc640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbe1c12b590 0x7fbe1c08fea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:33.498 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.495+0000 7fbe1b7fe640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbe1c12beb0 0x7fbe1c0903e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:33.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.495+0000 7fbe1a7fc640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbe1c12b590 0x7fbe1c08fea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:43180/0 (socket says 172.21.15.92:43180) 2024-09-21T09:43:33.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.495+0000 7fbe1affd640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbe1c096d20 0x7fbe1c08f960 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:44768/0 (socket says 172.21.15.92:44768) 2024-09-21T09:43:33.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.495+0000 7fbe1a7fc640 1 -- 172.21.15.92:0/569898063 learned_addr learned my addr 172.21.15.92:0/569898063 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:33.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.495+0000 7fbe1affd640 1 -- 172.21.15.92:0/569898063 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbe1c12beb0 msgr2=0x7fbe1c0903e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.495+0000 7fbe1affd640 1 --2- 172.21.15.92:0/569898063 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fbe1c12beb0 0x7fbe1c0903e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.495+0000 7fbe1affd640 1 -- 172.21.15.92:0/569898063 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbe1c12b590 msgr2=0x7fbe1c08fea0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.495+0000 7fbe1affd640 1 --2- 172.21.15.92:0/569898063 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fbe1c12b590 0x7fbe1c08fea0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.495+0000 7fbe1affd640 1 -- 172.21.15.92:0/569898063 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbe1c12e2c0 con 0x7fbe1c096d20 2024-09-21T09:43:33.499 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.495+0000 7fbe1affd640 1 --2- 172.21.15.92:0/569898063 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbe1c096d20 0x7fbe1c08f960 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7fbe0c00eb00 tx=0x7fbe0c00c2d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:33.500 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.495+0000 7fbe037fe640 1 -- 172.21.15.92:0/569898063 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbe0c010430 con 0x7fbe1c096d20 2024-09-21T09:43:33.500 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.495+0000 7fbe037fe640 1 -- 172.21.15.92:0/569898063 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fbe0c0105d0 con 0x7fbe1c096d20 2024-09-21T09:43:33.500 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.495+0000 7fbe037fe640 1 -- 172.21.15.92:0/569898063 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbe0c0109a0 con 0x7fbe1c096d20 2024-09-21T09:43:33.500 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.495+0000 7fbe216b1640 1 -- 172.21.15.92:0/569898063 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbe1c12d710 con 0x7fbe1c096d20 2024-09-21T09:43:33.500 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.496+0000 7fbe216b1640 1 -- 172.21.15.92:0/569898063 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fbe1c1c4960 con 0x7fbe1c096d20 2024-09-21T09:43:33.500 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.496+0000 7fbe216b1640 1 -- 172.21.15.92:0/569898063 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_get_version(what=osdmap handle=1) -- 0x7fbe1c098e70 con 0x7fbe1c096d20 2024-09-21T09:43:33.501 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.496+0000 7fbe037fe640 1 -- 172.21.15.92:0/569898063 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7fbe0c010b40 con 0x7fbe1c096d20 2024-09-21T09:43:33.502 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.497+0000 7fbe037fe640 1 -- 172.21.15.92:0/569898063 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fbe0c069ea0 con 0x7fbe1c096d20 2024-09-21T09:43:33.502 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.497+0000 7fbe037fe640 1 -- 172.21.15.92:0/569898063 --> v1:172.21.15.92:6813/138232475 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7fbe0c01be50 con 0x7fbe08057d30 2024-09-21T09:43:33.502 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.497+0000 7fbe037fe640 1 -- 172.21.15.92:0/569898063 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7fbe0c06c5a0 con 0x7fbe1c096d20 2024-09-21T09:43:33.502 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.498+0000 7fbe037fe640 1 -- 172.21.15.92:0/569898063 <== osd.3 v1:172.21.15.92:6813/138232475 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7fbe0c01be50 con 0x7fbe08057d30 2024-09-21T09:43:33.512 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.510+0000 7fbe216b1640 1 -- 172.21.15.92:0/569898063 --> v1:172.21.15.92:6813/138232475 -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) -- 0x7fbe1c1c4ed0 con 0x7fbe08057d30 2024-09-21T09:43:33.512 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.510+0000 7fbe037fe640 1 -- 172.21.15.92:0/569898063 <== osd.3 v1:172.21.15.92:6813/138232475 2 ==== command_reply(tid 2: 0 ) ==== 8+0+565 (unknown 0 0 125856537) 0x7fbe1c1c4ed0 con 0x7fbe08057d30 2024-09-21T09:43:33.513 INFO:teuthology.orchestra.run.smithi092.stdout:{ 2024-09-21T09:43:33.513 INFO:teuthology.orchestra.run.smithi092.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2024-09-21T09:43:33.513 INFO:teuthology.orchestra.run.smithi092.stdout:} 2024-09-21T09:43:33.515 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.511+0000 7fbe216b1640 1 -- 172.21.15.92:0/569898063 >> v1:172.21.15.92:6813/138232475 conn(0x7fbe08057d30 legacy=0x7fbe0805a170 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.515 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.511+0000 7fbe216b1640 1 -- 172.21.15.92:0/569898063 >> v1:172.21.15.92:6800/2299163855 conn(0x7fbe080514b0 legacy=0x7fbe08053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.515 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.511+0000 7fbe216b1640 1 -- 172.21.15.92:0/569898063 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbe1c096d20 msgr2=0x7fbe1c08f960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.515 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.511+0000 7fbe216b1640 1 --2- 172.21.15.92:0/569898063 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fbe1c096d20 0x7fbe1c08f960 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7fbe0c00eb00 tx=0x7fbe0c00c2d0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.516 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.511+0000 7fbe1b7fe640 1 -- 172.21.15.92:0/569898063 reap_dead start 2024-09-21T09:43:33.516 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.511+0000 7fbe216b1640 1 -- 172.21.15.92:0/569898063 shutdown_connections 2024-09-21T09:43:33.516 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.511+0000 7fbe216b1640 1 -- 172.21.15.92:0/569898063 >> 172.21.15.92:0/569898063 conn(0x7fbe1c09a3f0 msgr2=0x7fbe1c12a5b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:33.516 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.511+0000 7fbe216b1640 1 -- 172.21.15.92:0/569898063 shutdown_connections 2024-09-21T09:43:33.516 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.511+0000 7fbe216b1640 1 -- 172.21.15.92:0/569898063 wait complete. 2024-09-21T09:43:33.523 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 3 2024-09-21T09:43:33.628 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.622+0000 7fedef245640 1 -- 172.21.15.92:0/4175090995 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fede812f1c0 msgr2=0x7fede812f650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.628 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.622+0000 7fedef245640 1 --2- 172.21.15.92:0/4175090995 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fede812f1c0 0x7fede812f650 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7fede0009be0 tx=0x7fede002a770 comp rx=0 tx=0).stop 2024-09-21T09:43:33.628 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.622+0000 7fedef245640 1 -- 172.21.15.92:0/4175090995 shutdown_connections 2024-09-21T09:43:33.629 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.622+0000 7fedef245640 1 --2- 172.21.15.92:0/4175090995 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fede812f1c0 0x7fede812f650 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.629 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.622+0000 7fedef245640 1 --2- 172.21.15.92:0/4175090995 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fede8098d40 0x7fede8099120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.629 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.622+0000 7fedef245640 1 -- 172.21.15.92:0/4175090995 >> 172.21.15.92:0/4175090995 conn(0x7fede80a1790 msgr2=0x7fede80a1ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:33.629 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.622+0000 7fedef245640 1 -- 172.21.15.92:0/4175090995 shutdown_connections 2024-09-21T09:43:33.629 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.622+0000 7fedef245640 1 -- 172.21.15.92:0/4175090995 wait complete. 2024-09-21T09:43:33.629 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.623+0000 7fedef245640 1 Processor -- start 2024-09-21T09:43:33.630 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.623+0000 7fedef245640 1 -- start start 2024-09-21T09:43:33.630 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.623+0000 7fedef245640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fede8098d40 0x7fede80712c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:33.630 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.623+0000 7fedef245640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fede8099660 0x7fede8071800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:33.630 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.623+0000 7fedef245640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fede812f1c0 0x7fede8071d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:33.630 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.623+0000 7fedef245640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fede8137960 con 0x7fede812f1c0 2024-09-21T09:43:33.630 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.623+0000 7fedef245640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fede81377e0 con 0x7fede8098d40 2024-09-21T09:43:33.630 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.623+0000 7fedef245640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fede8137ae0 con 0x7fede8099660 2024-09-21T09:43:33.631 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.623+0000 7fedecfe1640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fede8098d40 0x7fede80712c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:33.631 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.623+0000 7fedecfe1640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fede8098d40 0x7fede80712c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:44792/0 (socket says 172.21.15.92:44792) 2024-09-21T09:43:33.631 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.623+0000 7fedecfe1640 1 -- 172.21.15.92:0/2965399418 learned_addr learned my addr 172.21.15.92:0/2965399418 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:33.631 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.623+0000 7fede7fff640 1 --2- 172.21.15.92:0/2965399418 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fede8099660 0x7fede8071800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:33.631 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.623+0000 7feded7e2640 1 --2- 172.21.15.92:0/2965399418 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fede812f1c0 0x7fede8071d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:33.631 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.624+0000 7fedecfe1640 1 -- 172.21.15.92:0/2965399418 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fede8099660 msgr2=0x7fede8071800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.632 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.624+0000 7fedecfe1640 1 --2- 172.21.15.92:0/2965399418 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fede8099660 0x7fede8071800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.632 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.624+0000 7fedecfe1640 1 -- 172.21.15.92:0/2965399418 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fede812f1c0 msgr2=0x7fede8071d40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.632 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.624+0000 7fedecfe1640 1 --2- 172.21.15.92:0/2965399418 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fede812f1c0 0x7fede8071d40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.632 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.624+0000 7fedecfe1640 1 -- 172.21.15.92:0/2965399418 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fede80730c0 con 0x7fede8098d40 2024-09-21T09:43:33.632 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.624+0000 7fede7fff640 1 --2- 172.21.15.92:0/2965399418 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fede8099660 0x7fede8071800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:33.632 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.624+0000 7feded7e2640 1 --2- 172.21.15.92:0/2965399418 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fede812f1c0 0x7fede8071d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:33.632 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.624+0000 7fedecfe1640 1 --2- 172.21.15.92:0/2965399418 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fede8098d40 0x7fede80712c0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7feddc00ea20 tx=0x7feddc00eee0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:33.632 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.624+0000 7fede57fa640 1 -- 172.21.15.92:0/2965399418 <== mon.1 v2:172.21.15.92:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feddc00ce40 con 0x7fede8098d40 2024-09-21T09:43:33.633 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.624+0000 7fede57fa640 1 -- 172.21.15.92:0/2965399418 <== mon.1 v2:172.21.15.92:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7feddc004020 con 0x7fede8098d40 2024-09-21T09:43:33.633 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.624+0000 7fede57fa640 1 -- 172.21.15.92:0/2965399418 <== mon.1 v2:172.21.15.92:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feddc004470 con 0x7fede8098d40 2024-09-21T09:43:33.633 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.624+0000 7fedef245640 1 -- 172.21.15.92:0/2965399418 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fede8072280 con 0x7fede8098d40 2024-09-21T09:43:33.633 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.624+0000 7fedef245640 1 -- 172.21.15.92:0/2965399418 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fede8073ca0 con 0x7fede8098d40 2024-09-21T09:43:33.633 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.625+0000 7fedef245640 1 -- 172.21.15.92:0/2965399418 --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fede8099120 con 0x7fede8098d40 2024-09-21T09:43:33.633 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.625+0000 7fede57fa640 1 -- 172.21.15.92:0/2965399418 <== mon.1 v2:172.21.15.92:3301/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7feddc0035f0 con 0x7fede8098d40 2024-09-21T09:43:33.633 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.626+0000 7fede57fa640 1 -- 172.21.15.92:0/2965399418 <== mon.1 v2:172.21.15.92:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7feddc069930 con 0x7fede8098d40 2024-09-21T09:43:33.634 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.629+0000 7fede57fa640 1 -- 172.21.15.92:0/2965399418 <== mon.1 v2:172.21.15.92:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7feddc009490 con 0x7fede8098d40 2024-09-21T09:43:33.781 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.779+0000 7fedef245640 1 -- 172.21.15.92:0/2965399418 --> v1:172.21.15.92:6800/2299163855 -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "3", "target": ["mon-mgr", ""]}) -- 0x7fede80698c0 con 0x7fedcc0514b0 2024-09-21T09:43:33.782 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.779+0000 7fede57fa640 1 -- 172.21.15.92:0/2965399418 <== mgr.14102 v1:172.21.15.92:6800/2299163855 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 3 to deep-scrub) ==== 41+0+0 (unknown 3526570807 0 0) 0x7fede80698c0 con 0x7fedcc0514b0 2024-09-21T09:43:33.782 INFO:teuthology.orchestra.run.smithi092.stderr:instructed osd(s) 3 to deep-scrub 2024-09-21T09:43:33.783 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.781+0000 7fedef245640 1 -- 172.21.15.92:0/2965399418 >> v1:172.21.15.92:6800/2299163855 conn(0x7fedcc0514b0 legacy=0x7fedcc053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.784 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.781+0000 7fedef245640 1 -- 172.21.15.92:0/2965399418 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fede8098d40 msgr2=0x7fede80712c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.784 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.781+0000 7fedef245640 1 --2- 172.21.15.92:0/2965399418 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fede8098d40 0x7fede80712c0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7feddc00ea20 tx=0x7feddc00eee0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.784 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.781+0000 7fedef245640 1 -- 172.21.15.92:0/2965399418 shutdown_connections 2024-09-21T09:43:33.784 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.781+0000 7fedef245640 1 --2- 172.21.15.92:0/2965399418 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fede812f1c0 0x7fede8071d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.784 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.781+0000 7fedef245640 1 --2- 172.21.15.92:0/2965399418 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fede8099660 0x7fede8071800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.784 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.781+0000 7fedef245640 1 --2- 172.21.15.92:0/2965399418 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fede8098d40 0x7fede80712c0 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.784 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.781+0000 7fedef245640 1 -- 172.21.15.92:0/2965399418 >> 172.21.15.92:0/2965399418 conn(0x7fede80a1790 msgr2=0x7fede809cfb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:33.784 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.781+0000 7fedef245640 1 -- 172.21.15.92:0/2965399418 shutdown_connections 2024-09-21T09:43:33.785 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.781+0000 7fedef245640 1 -- 172.21.15.92:0/2965399418 wait complete. 2024-09-21T09:43:33.794 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T09:43:33.893 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.891+0000 7ff5ea41d640 1 -- 172.21.15.92:0/3275057283 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff5e4131030 msgr2=0x7ff5e4133420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.893 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.891+0000 7ff5ea41d640 1 --2- 172.21.15.92:0/3275057283 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff5e4131030 0x7ff5e4133420 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7ff5dc009be0 tx=0x7ff5dc02a770 comp rx=0 tx=0).stop 2024-09-21T09:43:33.894 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.891+0000 7ff5ea41d640 1 -- 172.21.15.92:0/3275057283 shutdown_connections 2024-09-21T09:43:33.894 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.891+0000 7ff5ea41d640 1 --2- 172.21.15.92:0/3275057283 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff5e4131030 0x7ff5e4133420 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.894 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.891+0000 7ff5ea41d640 1 --2- 172.21.15.92:0/3275057283 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff5e412c330 0x7ff5e4130af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.894 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.891+0000 7ff5ea41d640 1 -- 172.21.15.92:0/3275057283 >> 172.21.15.92:0/3275057283 conn(0x7ff5e409a2e0 msgr2=0x7ff5e4097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:33.894 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.891+0000 7ff5ea41d640 1 -- 172.21.15.92:0/3275057283 shutdown_connections 2024-09-21T09:43:33.894 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.891+0000 7ff5ea41d640 1 -- 172.21.15.92:0/3275057283 wait complete. 2024-09-21T09:43:33.897 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.892+0000 7ff5ea41d640 1 Processor -- start 2024-09-21T09:43:33.897 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.892+0000 7ff5ea41d640 1 -- start start 2024-09-21T09:43:33.897 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.893+0000 7ff5ea41d640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff5e40991f0 0x7ff5e40945d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:33.897 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.893+0000 7ff5ea41d640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff5e412c330 0x7ff5e4094b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:33.897 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.893+0000 7ff5ea41d640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff5e4131030 0x7ff5e4095050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:33.897 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.893+0000 7ff5ea41d640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7ff5e4134750 con 0x7ff5e40991f0 2024-09-21T09:43:33.897 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.893+0000 7ff5ea41d640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7ff5e41345d0 con 0x7ff5e4131030 2024-09-21T09:43:33.897 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.893+0000 7ff5ea41d640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7ff5e41348d0 con 0x7ff5e412c330 2024-09-21T09:43:33.897 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.893+0000 7ff5e3fff640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff5e40991f0 0x7ff5e40945d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:33.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.893+0000 7ff5e37fe640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff5e412c330 0x7ff5e4094b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:33.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.893+0000 7ff5e3fff640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff5e40991f0 0x7ff5e40945d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:43202/0 (socket says 172.21.15.92:43202) 2024-09-21T09:43:33.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.893+0000 7ff5e3fff640 1 -- 172.21.15.92:0/2612943374 learned_addr learned my addr 172.21.15.92:0/2612943374 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:33.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.893+0000 7ff5e89ba640 1 --2- 172.21.15.92:0/2612943374 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff5e4131030 0x7ff5e4095050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:33.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.893+0000 7ff5e37fe640 1 -- 172.21.15.92:0/2612943374 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff5e4131030 msgr2=0x7ff5e4095050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.893+0000 7ff5e37fe640 1 --2- 172.21.15.92:0/2612943374 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff5e4131030 0x7ff5e4095050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.893+0000 7ff5e37fe640 1 -- 172.21.15.92:0/2612943374 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff5e40991f0 msgr2=0x7ff5e40945d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:33.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.893+0000 7ff5e37fe640 1 --2- 172.21.15.92:0/2612943374 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff5e40991f0 0x7ff5e40945d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:33.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.893+0000 7ff5e37fe640 1 -- 172.21.15.92:0/2612943374 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff5e4094030 con 0x7ff5e412c330 2024-09-21T09:43:33.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.893+0000 7ff5e37fe640 1 --2- 172.21.15.92:0/2612943374 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff5e412c330 0x7ff5e4094b10 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7ff5d800e9f0 tx=0x7ff5d800eeb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:33.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.893+0000 7ff5e0ff9640 1 -- 172.21.15.92:0/2612943374 <== mon.2 v2:172.21.15.92:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff5d800cd80 con 0x7ff5e412c330 2024-09-21T09:43:33.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.893+0000 7ff5e0ff9640 1 -- 172.21.15.92:0/2612943374 <== mon.2 v2:172.21.15.92:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7ff5d8004020 con 0x7ff5e412c330 2024-09-21T09:43:33.898 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.893+0000 7ff5ea41d640 1 -- 172.21.15.92:0/2612943374 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff5e408f300 con 0x7ff5e412c330 2024-09-21T09:43:33.899 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.893+0000 7ff5e0ff9640 1 -- 172.21.15.92:0/2612943374 <== mon.2 v2:172.21.15.92:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff5d8004450 con 0x7ff5e412c330 2024-09-21T09:43:33.899 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.893+0000 7ff5ea41d640 1 -- 172.21.15.92:0/2612943374 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7ff5e4130470 con 0x7ff5e412c330 2024-09-21T09:43:33.899 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.894+0000 7ff5ea41d640 1 -- 172.21.15.92:0/2612943374 --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff5e412bc20 con 0x7ff5e412c330 2024-09-21T09:43:33.899 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.894+0000 7ff5e0ff9640 1 -- 172.21.15.92:0/2612943374 <== mon.2 v2:172.21.15.92:3302/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7ff5d80035f0 con 0x7ff5e412c330 2024-09-21T09:43:33.899 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.895+0000 7ff5e0ff9640 1 -- 172.21.15.92:0/2612943374 <== mon.2 v2:172.21.15.92:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7ff5d80698d0 con 0x7ff5e412c330 2024-09-21T09:43:33.900 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:33.898+0000 7ff5e0ff9640 1 -- 172.21.15.92:0/2612943374 <== mon.2 v2:172.21.15.92:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7ff5d8068980 con 0x7ff5e412c330 2024-09-21T09:43:34.044 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:34.043+0000 7ff5ea41d640 1 -- 172.21.15.92:0/2612943374 --> v1:172.21.15.92:6800/2299163855 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7ff5e408f5e0 con 0x7ff5d00514b0 2024-09-21T09:43:34.045 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:34.043+0000 7ff5e0ff9640 1 -- 172.21.15.92:0/2612943374 <== mgr.14102 v1:172.21.15.92:6800/2299163855 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32001 (unknown 2965378022 0 611775025) 0x7ff5d40012d0 con 0x7ff5d00514b0 2024-09-21T09:43:34.045 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:34.046 INFO:teuthology.orchestra.run.smithi092.stderr:dumped all 2024-09-21T09:43:34.047 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:34.045+0000 7ff5ea41d640 1 -- 172.21.15.92:0/2612943374 >> v1:172.21.15.92:6800/2299163855 conn(0x7ff5d00514b0 legacy=0x7ff5d0053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:34.047 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:34.045+0000 7ff5ea41d640 1 -- 172.21.15.92:0/2612943374 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff5e412c330 msgr2=0x7ff5e4094b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:34.047 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:34.045+0000 7ff5ea41d640 1 --2- 172.21.15.92:0/2612943374 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff5e412c330 0x7ff5e4094b10 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7ff5d800e9f0 tx=0x7ff5d800eeb0 comp rx=0 tx=0).stop 2024-09-21T09:43:34.047 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:34.045+0000 7ff5ea41d640 1 -- 172.21.15.92:0/2612943374 shutdown_connections 2024-09-21T09:43:34.047 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:34.045+0000 7ff5ea41d640 1 --2- 172.21.15.92:0/2612943374 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7ff5e4131030 0x7ff5e4095050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:34.047 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:34.045+0000 7ff5ea41d640 1 --2- 172.21.15.92:0/2612943374 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7ff5e412c330 0x7ff5e4094b10 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:34.047 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:34.045+0000 7ff5ea41d640 1 --2- 172.21.15.92:0/2612943374 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7ff5e40991f0 0x7ff5e40945d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:34.047 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:34.045+0000 7ff5ea41d640 1 -- 172.21.15.92:0/2612943374 >> 172.21.15.92:0/2612943374 conn(0x7ff5e409a2e0 msgr2=0x7ff5e409c810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:34.048 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:34.045+0000 7ff5ea41d640 1 -- 172.21.15.92:0/2612943374 shutdown_connections 2024-09-21T09:43:34.048 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:34.045+0000 7ff5ea41d640 1 -- 172.21.15.92:0/2612943374 wait complete. 2024-09-21T09:43:34.058 INFO:teuthology.orchestra.run.smithi092.stdout:{"pg_ready":true,"pg_map":{"version":28,"stamp":"2024-09-21T09:43:32.276238+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112300,"kb_used_data":2256,"kb_used_omap":17,"kb_used_meta":109934,"kb_avail":377375060,"statfs":{"total":386547056640,"available":386432061440,"internally_reserved":0,"allocated":2310144,"data_stored":1565810,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17726,"internal_metadata":112573122},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001251"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":52,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350236+0000","last_change":"2024-09-21T09:43:21.330614+0000","last_active":"2024-09-21T09:43:26.350236+0000","last_peered":"2024-09-21T09:43:26.350236+0000","last_clean":"2024-09-21T09:43:26.350236+0000","last_became_active":"2024-09-21T09:43:21.330325+0000","last_became_peered":"2024-09-21T09:43:21.330325+0000","last_unstale":"2024-09-21T09:43:26.350236+0000","last_undegraded":"2024-09-21T09:43:26.350236+0000","last_fullsized":"2024-09-21T09:43:26.350236+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:17:00.614617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084231599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":42,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.720522+0000","last_change":"2024-09-21T09:43:19.309958+0000","last_active":"2024-09-21T09:43:26.720522+0000","last_peered":"2024-09-21T09:43:26.720522+0000","last_clean":"2024-09-21T09:43:26.720522+0000","last_became_active":"2024-09-21T09:43:19.309603+0000","last_became_peered":"2024-09-21T09:43:19.309603+0000","last_unstale":"2024-09-21T09:43:26.720522+0000","last_undegraded":"2024-09-21T09:43:26.720522+0000","last_fullsized":"2024-09-21T09:43:26.720522+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:26:08.037398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049204800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":52,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350258+0000","last_change":"2024-09-21T09:43:21.330875+0000","last_active":"2024-09-21T09:43:26.350258+0000","last_peered":"2024-09-21T09:43:26.350258+0000","last_clean":"2024-09-21T09:43:26.350258+0000","last_became_active":"2024-09-21T09:43:21.330619+0000","last_became_peered":"2024-09-21T09:43:21.330619+0000","last_unstale":"2024-09-21T09:43:26.350258+0000","last_undegraded":"2024-09-21T09:43:26.350258+0000","last_fullsized":"2024-09-21T09:43:26.350258+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:22:03.959366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080402800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":42,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.720283+0000","last_change":"2024-09-21T09:43:19.310001+0000","last_active":"2024-09-21T09:43:26.720283+0000","last_peered":"2024-09-21T09:43:26.720283+0000","last_clean":"2024-09-21T09:43:26.720283+0000","last_became_active":"2024-09-21T09:43:19.309659+0000","last_became_peered":"2024-09-21T09:43:19.309659+0000","last_unstale":"2024-09-21T09:43:26.720283+0000","last_undegraded":"2024-09-21T09:43:26.720283+0000","last_fullsized":"2024-09-21T09:43:26.720283+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:39.861259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000465849,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"18'2","reported_seq":50,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350048+0000","last_change":"2024-09-21T09:43:21.315323+0000","last_active":"2024-09-21T09:43:26.350048+0000","last_peered":"2024-09-21T09:43:26.350048+0000","last_clean":"2024-09-21T09:43:26.350048+0000","last_became_active":"2024-09-21T09:43:21.314821+0000","last_became_peered":"2024-09-21T09:43:21.314821+0000","last_unstale":"2024-09-21T09:43:26.350048+0000","last_undegraded":"2024-09-21T09:43:26.350048+0000","last_fullsized":"2024-09-21T09:43:26.350048+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:38:32.540545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010637209999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":39,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350506+0000","last_change":"2024-09-21T09:43:21.315433+0000","last_active":"2024-09-21T09:43:26.350506+0000","last_peered":"2024-09-21T09:43:26.350506+0000","last_clean":"2024-09-21T09:43:26.350506+0000","last_became_active":"2024-09-21T09:43:21.315100+0000","last_became_peered":"2024-09-21T09:43:21.315100+0000","last_unstale":"2024-09-21T09:43:26.350506+0000","last_undegraded":"2024-09-21T09:43:26.350506+0000","last_fullsized":"2024-09-21T09:43:26.350506+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:05:55.727692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049819100000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":48,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.349885+0000","last_change":"2024-09-21T09:43:21.315140+0000","last_active":"2024-09-21T09:43:26.349885+0000","last_peered":"2024-09-21T09:43:26.349885+0000","last_clean":"2024-09-21T09:43:26.349885+0000","last_became_active":"2024-09-21T09:43:21.314817+0000","last_became_peered":"2024-09-21T09:43:21.314817+0000","last_unstale":"2024-09-21T09:43:26.349885+0000","last_undegraded":"2024-09-21T09:43:26.349885+0000","last_fullsized":"2024-09-21T09:43:26.349885+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:08:25.622786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088008499999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"16'1","reported_seq":47,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.350078+0000","last_change":"2024-09-21T09:43:21.315433+0000","last_active":"2024-09-21T09:43:26.350078+0000","last_peered":"2024-09-21T09:43:26.350078+0000","last_clean":"2024-09-21T09:43:26.350078+0000","last_became_active":"2024-09-21T09:43:21.315110+0000","last_became_peered":"2024-09-21T09:43:21.315110+0000","last_unstale":"2024-09-21T09:43:26.350078+0000","last_undegraded":"2024-09-21T09:43:26.350078+0000","last_fullsized":"2024-09-21T09:43:26.350078+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:52.818234+0000","last_clean_scrub_stamp":"2024-09-21T09:41:52.818234+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:42.627708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010430319999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"21'104","reported_seq":195,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:26.720214+0000","last_change":"2024-09-21T09:43:19.310088+0000","last_active":"2024-09-21T09:43:26.720214+0000","last_peered":"2024-09-21T09:43:26.720214+0000","last_clean":"2024-09-21T09:43:26.720214+0000","last_became_active":"2024-09-21T09:43:19.309749+0000","last_became_peered":"2024-09-21T09:43:19.309749+0000","last_unstale":"2024-09-21T09:43:26.720214+0000","last_undegraded":"2024-09-21T09:43:26.720214+0000","last_fullsized":"2024-09-21T09:43:26.720214+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:41:50.808452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:41:50.808452+0000","last_clean_scrub_stamp":"2024-09-21T09:41:50.808452+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:18:22.448558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":30,"seq":128849018884,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27820,"kb_used_data":276,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344020,"statfs":{"total":96636764160,"available":96608276480,"internally_reserved":0,"allocated":282624,"data_stored":96278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4436,"internal_metadata":28176044},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":29,"seq":124554051588,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27816,"kb_used_data":272,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344024,"statfs":{"total":96636764160,"available":96608280576,"internally_reserved":0,"allocated":278528,"data_stored":96259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4432,"internal_metadata":28176048},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":28,"seq":120259084292,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28336,"kb_used_data":856,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343504,"statfs":{"total":96636764160,"available":96607748096,"internally_reserved":0,"allocated":876544,"data_stored":686646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4431,"internal_metadata":28110513},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":27,"seq":115964116997,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28328,"kb_used_data":852,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343512,"statfs":{"total":96636764160,"available":96607756288,"internally_reserved":0,"allocated":872448,"data_stored":686627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4427,"internal_metadata":28110517},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:43:34.063 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2024-09-21T09:41:52.818234+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=41, tm_sec=52, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=43, tm_sec=31, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T09:43:34.063 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2024-09-21T09:41:52.818234+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=41, tm_sec=52, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=43, tm_sec=31, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T09:43:34.063 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2024-09-21T09:41:52.818234+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=41, tm_sec=52, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=43, tm_sec=31, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T09:43:34.063 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2024-09-21T09:41:52.818234+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=41, tm_sec=52, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=43, tm_sec=31, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T09:43:34.064 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2024-09-21T09:41:52.818234+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=41, tm_sec=52, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=43, tm_sec=31, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T09:43:34.064 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2024-09-21T09:41:52.818234+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=41, tm_sec=52, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=43, tm_sec=31, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T09:43:34.064 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2024-09-21T09:41:52.818234+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=41, tm_sec=52, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=43, tm_sec=31, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T09:43:34.064 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2024-09-21T09:41:52.818234+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=41, tm_sec=52, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=43, tm_sec=31, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T09:43:34.064 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2024-09-21T09:41:50.808452+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=41, tm_sec=50, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=9, tm_min=43, tm_sec=31, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T09:43:34.065 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2024-09-21T09:43:54.066 DEBUG:teuthology.orchestra.run.smithi092:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T09:43:54.177 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.174+0000 7f6f86bc5640 1 -- 172.21.15.92:0/3934234444 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6f80131030 msgr2=0x7f6f80133420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:54.177 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.174+0000 7f6f86bc5640 1 --2- 172.21.15.92:0/3934234444 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6f80131030 0x7f6f80133420 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f6f78005fd0 tx=0x7f6f7802a780 comp rx=0 tx=0).stop 2024-09-21T09:43:54.177 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.174+0000 7f6f86bc5640 1 -- 172.21.15.92:0/3934234444 shutdown_connections 2024-09-21T09:43:54.177 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.174+0000 7f6f86bc5640 1 --2- 172.21.15.92:0/3934234444 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6f80131030 0x7f6f80133420 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:54.178 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.174+0000 7f6f86bc5640 1 --2- 172.21.15.92:0/3934234444 >> v2:172.21.15.92:3300/0 conn(0x7f6f8012c3d0 0x7f6f80130af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:54.178 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.174+0000 7f6f86bc5640 1 -- 172.21.15.92:0/3934234444 >> 172.21.15.92:0/3934234444 conn(0x7f6f8009a2e0 msgr2=0x7f6f80097ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:54.178 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.174+0000 7f6f86bc5640 1 -- 172.21.15.92:0/3934234444 shutdown_connections 2024-09-21T09:43:54.178 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.174+0000 7f6f86bc5640 1 -- 172.21.15.92:0/3934234444 wait complete. 2024-09-21T09:43:54.180 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.175+0000 7f6f86bc5640 1 Processor -- start 2024-09-21T09:43:54.180 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.175+0000 7f6f86bc5640 1 -- start start 2024-09-21T09:43:54.180 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.176+0000 7f6f86bc5640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6f800991f0 0x7f6f800945d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:54.180 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.176+0000 7f6f86bc5640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6f8012c3d0 0x7f6f80094b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:54.180 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.176+0000 7f6f86bc5640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6f80131030 0x7f6f80095050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:54.180 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.176+0000 7f6f86bc5640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7f6f801345e0 con 0x7f6f800991f0 2024-09-21T09:43:54.180 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.176+0000 7f6f86bc5640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7f6f80134460 con 0x7f6f80131030 2024-09-21T09:43:54.180 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.176+0000 7f6f86bc5640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7f6f80134760 con 0x7f6f8012c3d0 2024-09-21T09:43:54.180 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.176+0000 7f6f84961640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6f800991f0 0x7f6f800945d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:54.180 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.176+0000 7f6f85162640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6f80131030 0x7f6f80095050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:54.181 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.176+0000 7f6f7ffff640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6f8012c3d0 0x7f6f80094b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:54.181 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.176+0000 7f6f84961640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6f800991f0 0x7f6f800945d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:34540/0 (socket says 172.21.15.92:34540) 2024-09-21T09:43:54.181 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.176+0000 7f6f85162640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6f80131030 0x7f6f80095050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3301/0 says I am v2:172.21.15.92:42072/0 (socket says 172.21.15.92:42072) 2024-09-21T09:43:54.181 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.176+0000 7f6f84961640 1 -- 172.21.15.92:0/3349499514 learned_addr learned my addr 172.21.15.92:0/3349499514 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:54.181 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.176+0000 7f6f84961640 1 -- 172.21.15.92:0/3349499514 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6f8012c3d0 msgr2=0x7f6f80094b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:54.181 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.176+0000 7f6f84961640 1 --2- 172.21.15.92:0/3349499514 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6f8012c3d0 0x7f6f80094b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:54.181 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.176+0000 7f6f84961640 1 -- 172.21.15.92:0/3349499514 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6f80131030 msgr2=0x7f6f80095050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:54.181 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.176+0000 7f6f84961640 1 --2- 172.21.15.92:0/3349499514 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6f80131030 0x7f6f80095050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:54.181 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.176+0000 7f6f84961640 1 -- 172.21.15.92:0/3349499514 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6f8008e650 con 0x7f6f800991f0 2024-09-21T09:43:54.181 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.176+0000 7f6f85162640 1 --2- 172.21.15.92:0/3349499514 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6f80131030 0x7f6f80095050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:54.181 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.176+0000 7f6f7ffff640 1 --2- 172.21.15.92:0/3349499514 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6f8012c3d0 0x7f6f80094b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:54.181 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.176+0000 7f6f84961640 1 --2- 172.21.15.92:0/3349499514 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6f800991f0 0x7f6f800945d0 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f6f74001680 tx=0x7f6f74001b40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:54.181 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.176+0000 7f6f7d7fa640 1 -- 172.21.15.92:0/3349499514 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6f74001d30 con 0x7f6f800991f0 2024-09-21T09:43:54.182 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.176+0000 7f6f7d7fa640 1 -- 172.21.15.92:0/3349499514 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f6f74005070 con 0x7f6f800991f0 2024-09-21T09:43:54.182 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.177+0000 7f6f86bc5640 1 -- 172.21.15.92:0/3349499514 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6f80130d90 con 0x7f6f800991f0 2024-09-21T09:43:54.182 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.177+0000 7f6f7d7fa640 1 -- 172.21.15.92:0/3349499514 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6f74010630 con 0x7f6f800991f0 2024-09-21T09:43:54.182 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.177+0000 7f6f86bc5640 1 -- 172.21.15.92:0/3349499514 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f6f8008ecc0 con 0x7f6f800991f0 2024-09-21T09:43:54.182 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.178+0000 7f6f86bc5640 1 -- 172.21.15.92:0/3349499514 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6f8012bcc0 con 0x7f6f800991f0 2024-09-21T09:43:54.183 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.178+0000 7f6f7d7fa640 1 -- 172.21.15.92:0/3349499514 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7f6f740107d0 con 0x7f6f800991f0 2024-09-21T09:43:54.183 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.178+0000 7f6f7d7fa640 1 -- 172.21.15.92:0/3349499514 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f6f7406ce70 con 0x7f6f800991f0 2024-09-21T09:43:54.184 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.182+0000 7f6f7d7fa640 1 -- 172.21.15.92:0/3349499514 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f6f7401d030 con 0x7f6f800991f0 2024-09-21T09:43:54.333 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.330+0000 7f6f86bc5640 1 -- 172.21.15.92:0/3349499514 --> v1:172.21.15.92:6800/2299163855 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f6f800699c0 con 0x7f6f64051650 2024-09-21T09:43:54.334 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.331+0000 7f6f7d7fa640 1 -- 172.21.15.92:0/3349499514 <== mgr.14102 v1:172.21.15.92:6800/2299163855 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32012 (unknown 2965378022 0 819172917) 0x7f6f800699c0 con 0x7f6f64051650 2024-09-21T09:43:54.334 INFO:teuthology.orchestra.run.smithi092.stdout: 2024-09-21T09:43:54.334 INFO:teuthology.orchestra.run.smithi092.stderr:dumped all 2024-09-21T09:43:54.336 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.332+0000 7f6f86bc5640 1 -- 172.21.15.92:0/3349499514 >> v1:172.21.15.92:6800/2299163855 conn(0x7f6f64051650 legacy=0x7f6f64053b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:54.336 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.333+0000 7f6f86bc5640 1 -- 172.21.15.92:0/3349499514 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6f800991f0 msgr2=0x7f6f800945d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:54.336 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.333+0000 7f6f86bc5640 1 --2- 172.21.15.92:0/3349499514 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6f800991f0 0x7f6f800945d0 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 crypto rx=0x7f6f74001680 tx=0x7f6f74001b40 comp rx=0 tx=0).stop 2024-09-21T09:43:54.336 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.333+0000 7f6f86bc5640 1 -- 172.21.15.92:0/3349499514 shutdown_connections 2024-09-21T09:43:54.336 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.333+0000 7f6f86bc5640 1 --2- 172.21.15.92:0/3349499514 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7f6f80131030 0x7f6f80095050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:54.336 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.333+0000 7f6f86bc5640 1 --2- 172.21.15.92:0/3349499514 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7f6f8012c3d0 0x7f6f80094b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:54.336 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.333+0000 7f6f86bc5640 1 --2- 172.21.15.92:0/3349499514 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7f6f800991f0 0x7f6f800945d0 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:54.336 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.333+0000 7f6f86bc5640 1 -- 172.21.15.92:0/3349499514 >> 172.21.15.92:0/3349499514 conn(0x7f6f8009a2e0 msgr2=0x7f6f8009bc40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:54.336 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.333+0000 7f6f86bc5640 1 -- 172.21.15.92:0/3349499514 shutdown_connections 2024-09-21T09:43:54.336 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.333+0000 7f6f86bc5640 1 -- 172.21.15.92:0/3349499514 wait complete. 2024-09-21T09:43:54.346 INFO:teuthology.orchestra.run.smithi092.stdout:{"pg_ready":true,"pg_map":{"version":39,"stamp":"2024-09-21T09:43:54.280078+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110,"ondisk_log_size":110,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112272,"kb_used_data":2256,"kb_used_omap":17,"kb_used_meta":109934,"kb_avail":377375088,"statfs":{"total":386547056640,"available":386432090112,"internally_reserved":0,"allocated":2310144,"data_stored":1565810,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17726,"internal_metadata":112573122},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002093"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":60,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:37.586491+0000","last_change":"2024-09-21T09:43:37.586491+0000","last_active":"2024-09-21T09:43:37.586491+0000","last_peered":"2024-09-21T09:43:37.586491+0000","last_clean":"2024-09-21T09:43:37.586491+0000","last_became_active":"2024-09-21T09:43:21.330325+0000","last_became_peered":"2024-09-21T09:43:21.330325+0000","last_unstale":"2024-09-21T09:43:37.586491+0000","last_undegraded":"2024-09-21T09:43:37.586491+0000","last_fullsized":"2024-09-21T09:43:37.586491+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:43:37.586394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:43:37.586394+0000","last_clean_scrub_stamp":"2024-09-21T09:43:37.586394+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:39:21.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084231599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":50,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:34.091194+0000","last_change":"2024-09-21T09:43:34.091194+0000","last_active":"2024-09-21T09:43:34.091194+0000","last_peered":"2024-09-21T09:43:34.091194+0000","last_clean":"2024-09-21T09:43:34.091194+0000","last_became_active":"2024-09-21T09:43:19.309603+0000","last_became_peered":"2024-09-21T09:43:19.309603+0000","last_unstale":"2024-09-21T09:43:34.091194+0000","last_undegraded":"2024-09-21T09:43:34.091194+0000","last_fullsized":"2024-09-21T09:43:34.091194+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:43:34.091094+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:43:34.091094+0000","last_clean_scrub_stamp":"2024-09-21T09:43:34.091094+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:53:24.945473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049204800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":60,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:35.546800+0000","last_change":"2024-09-21T09:43:35.546800+0000","last_active":"2024-09-21T09:43:35.546800+0000","last_peered":"2024-09-21T09:43:35.546800+0000","last_clean":"2024-09-21T09:43:35.546800+0000","last_became_active":"2024-09-21T09:43:21.330619+0000","last_became_peered":"2024-09-21T09:43:21.330619+0000","last_unstale":"2024-09-21T09:43:35.546800+0000","last_undegraded":"2024-09-21T09:43:35.546800+0000","last_fullsized":"2024-09-21T09:43:35.546800+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:43:35.546702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:43:35.546702+0000","last_clean_scrub_stamp":"2024-09-21T09:43:35.546702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:14:49.713998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080402800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":50,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:33.123423+0000","last_change":"2024-09-21T09:43:33.123423+0000","last_active":"2024-09-21T09:43:33.123423+0000","last_peered":"2024-09-21T09:43:33.123423+0000","last_clean":"2024-09-21T09:43:33.123423+0000","last_became_active":"2024-09-21T09:43:19.309659+0000","last_became_peered":"2024-09-21T09:43:19.309659+0000","last_unstale":"2024-09-21T09:43:33.123423+0000","last_undegraded":"2024-09-21T09:43:33.123423+0000","last_fullsized":"2024-09-21T09:43:33.123423+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:43:33.123342+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:43:33.123342+0000","last_clean_scrub_stamp":"2024-09-21T09:43:33.123342+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:49:42.497968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000465849,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"33'3","reported_seq":59,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:38.551811+0000","last_change":"2024-09-21T09:43:38.551811+0000","last_active":"2024-09-21T09:43:38.551811+0000","last_peered":"2024-09-21T09:43:38.551811+0000","last_clean":"2024-09-21T09:43:38.551811+0000","last_became_active":"2024-09-21T09:43:21.314821+0000","last_became_peered":"2024-09-21T09:43:21.314821+0000","last_unstale":"2024-09-21T09:43:38.551811+0000","last_undegraded":"2024-09-21T09:43:38.551811+0000","last_fullsized":"2024-09-21T09:43:38.551811+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"33'3","last_scrub_stamp":"2024-09-21T09:43:38.551691+0000","last_deep_scrub":"33'3","last_deep_scrub_stamp":"2024-09-21T09:43:38.551691+0000","last_clean_scrub_stamp":"2024-09-21T09:43:38.551691+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:09:03.029418+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.0010637209999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":47,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:33.514168+0000","last_change":"2024-09-21T09:43:33.514168+0000","last_active":"2024-09-21T09:43:33.514168+0000","last_peered":"2024-09-21T09:43:33.514168+0000","last_clean":"2024-09-21T09:43:33.514168+0000","last_became_active":"2024-09-21T09:43:21.315100+0000","last_became_peered":"2024-09-21T09:43:21.315100+0000","last_unstale":"2024-09-21T09:43:33.514168+0000","last_undegraded":"2024-09-21T09:43:33.514168+0000","last_fullsized":"2024-09-21T09:43:33.514168+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:43:33.514063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:43:33.514063+0000","last_clean_scrub_stamp":"2024-09-21T09:43:33.514063+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:24:05.553978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049819100000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":56,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:34.573195+0000","last_change":"2024-09-21T09:43:34.573195+0000","last_active":"2024-09-21T09:43:34.573195+0000","last_peered":"2024-09-21T09:43:34.573195+0000","last_clean":"2024-09-21T09:43:34.573195+0000","last_became_active":"2024-09-21T09:43:21.314817+0000","last_became_peered":"2024-09-21T09:43:21.314817+0000","last_unstale":"2024-09-21T09:43:34.573195+0000","last_undegraded":"2024-09-21T09:43:34.573195+0000","last_fullsized":"2024-09-21T09:43:34.573195+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:43:34.573122+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:43:34.573122+0000","last_clean_scrub_stamp":"2024-09-21T09:43:34.573122+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:51:14.244449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00088008499999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"33'2","reported_seq":56,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:36.567240+0000","last_change":"2024-09-21T09:43:36.567240+0000","last_active":"2024-09-21T09:43:36.567240+0000","last_peered":"2024-09-21T09:43:36.567240+0000","last_clean":"2024-09-21T09:43:36.567240+0000","last_became_active":"2024-09-21T09:43:21.315110+0000","last_became_peered":"2024-09-21T09:43:21.315110+0000","last_unstale":"2024-09-21T09:43:36.567240+0000","last_undegraded":"2024-09-21T09:43:36.567240+0000","last_fullsized":"2024-09-21T09:43:36.567240+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":15,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"33'2","last_scrub_stamp":"2024-09-21T09:43:36.567124+0000","last_deep_scrub":"33'2","last_deep_scrub_stamp":"2024-09-21T09:43:36.567124+0000","last_clean_scrub_stamp":"2024-09-21T09:43:36.567124+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:29:28.577945+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0.0010430319999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"33'105","reported_seq":204,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T09:43:35.131480+0000","last_change":"2024-09-21T09:43:35.131480+0000","last_active":"2024-09-21T09:43:35.131480+0000","last_peered":"2024-09-21T09:43:35.131480+0000","last_clean":"2024-09-21T09:43:35.131480+0000","last_became_active":"2024-09-21T09:43:19.309749+0000","last_became_peered":"2024-09-21T09:43:19.309749+0000","last_unstale":"2024-09-21T09:43:35.131480+0000","last_undegraded":"2024-09-21T09:43:35.131480+0000","last_fullsized":"2024-09-21T09:43:35.131480+0000","mapping_epoch":28,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":29,"parent":"0.0","parent_split_bits":0,"last_scrub":"33'105","last_scrub_stamp":"2024-09-21T09:43:35.131407+0000","last_deep_scrub":"33'105","last_deep_scrub_stamp":"2024-09-21T09:43:35.131407+0000","last_clean_scrub_stamp":"2024-09-21T09:43:35.131407+0000","objects_scrubbed":2,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:26:38.240252+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105,"ondisk_log_size":105,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":30,"seq":128849018888,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27812,"kb_used_data":276,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344028,"statfs":{"total":96636764160,"available":96608284672,"internally_reserved":0,"allocated":282624,"data_stored":96278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4436,"internal_metadata":28176044},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":29,"seq":124554051593,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27808,"kb_used_data":272,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344032,"statfs":{"total":96636764160,"available":96608288768,"internally_reserved":0,"allocated":278528,"data_stored":96259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4432,"internal_metadata":28176048},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":28,"seq":120259084297,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28328,"kb_used_data":856,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343512,"statfs":{"total":96636764160,"available":96607756288,"internally_reserved":0,"allocated":876544,"data_stored":686646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4431,"internal_metadata":28110513},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":27,"seq":115964117001,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28324,"kb_used_data":852,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343516,"statfs":{"total":96636764160,"available":96607760384,"internally_reserved":0,"allocated":872448,"data_stored":686627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4427,"internal_metadata":28110517},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:43:54.347 DEBUG:teuthology.orchestra.run.smithi092:> sudo ceph --cluster ceph config set global mon_health_to_clog false 2024-09-21T09:43:54.448 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.445+0000 7fe2dcfea640 1 -- 172.21.15.92:0/2957338401 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe2d8131fa0 msgr2=0x7fe2d8134390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:54.448 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.445+0000 7fe2dcfea640 1 --2- 172.21.15.92:0/2957338401 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe2d8131fa0 0x7fe2d8134390 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 crypto rx=0x7fe2c0009860 tx=0x7fe2c002a420 comp rx=0 tx=0).stop 2024-09-21T09:43:54.448 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.445+0000 7fe2dcfea640 1 -- 172.21.15.92:0/2957338401 shutdown_connections 2024-09-21T09:43:54.448 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.445+0000 7fe2dcfea640 1 --2- 172.21.15.92:0/2957338401 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe2d81348d0 0x7fe2d8136cc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:54.448 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.445+0000 7fe2dcfea640 1 --2- 172.21.15.92:0/2957338401 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe2d8131fa0 0x7fe2d8134390 unknown :-1 s=CLOSED pgs=141 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:54.448 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.445+0000 7fe2dcfea640 1 -- 172.21.15.92:0/2957338401 >> 172.21.15.92:0/2957338401 conn(0x7fe2d80a1970 msgr2=0x7fe2d80a1d80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:54.448 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.445+0000 7fe2dcfea640 1 -- 172.21.15.92:0/2957338401 shutdown_connections 2024-09-21T09:43:54.448 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.445+0000 7fe2dcfea640 1 -- 172.21.15.92:0/2957338401 wait complete. 2024-09-21T09:43:54.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.446+0000 7fe2dcfea640 1 Processor -- start 2024-09-21T09:43:54.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.446+0000 7fe2dcfea640 1 -- start start 2024-09-21T09:43:54.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.446+0000 7fe2dcfea640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe2d8099200 0x7fe2d80954e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:54.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.446+0000 7fe2dcfea640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe2d8131fa0 0x7fe2d8095a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:54.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.446+0000 7fe2dcfea640 1 --2- >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe2d81348d0 0x7fe2d8095f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T09:43:54.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.446+0000 7fe2dcfea640 1 -- --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_getmap magic: 0 -- 0x7fe2d8137af0 con 0x7fe2d8099200 2024-09-21T09:43:54.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.446+0000 7fe2dcfea640 1 -- --> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] -- mon_getmap magic: 0 -- 0x7fe2d8137970 con 0x7fe2d8131fa0 2024-09-21T09:43:54.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.446+0000 7fe2dcfea640 1 -- --> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] -- mon_getmap magic: 0 -- 0x7fe2d8137c70 con 0x7fe2d81348d0 2024-09-21T09:43:54.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.446+0000 7fe2d659c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe2d8099200 0x7fe2d80954e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:54.451 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.446+0000 7fe2d5d9b640 1 --2- >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe2d8131fa0 0x7fe2d8095a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:54.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.446+0000 7fe2d659c640 1 --2- >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe2d8099200 0x7fe2d80954e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.92:3300/0 says I am v2:172.21.15.92:34552/0 (socket says 172.21.15.92:34552) 2024-09-21T09:43:54.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.446+0000 7fe2d659c640 1 -- 172.21.15.92:0/2277478156 learned_addr learned my addr 172.21.15.92:0/2277478156 (peer_addr_for_me v2:172.21.15.92:0/0) 2024-09-21T09:43:54.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.446+0000 7fe2d6d9d640 1 --2- 172.21.15.92:0/2277478156 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe2d81348d0 0x7fe2d8095f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T09:43:54.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.447+0000 7fe2d659c640 1 -- 172.21.15.92:0/2277478156 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe2d81348d0 msgr2=0x7fe2d8095f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:54.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.447+0000 7fe2d659c640 1 --2- 172.21.15.92:0/2277478156 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe2d81348d0 0x7fe2d8095f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:54.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.447+0000 7fe2d659c640 1 -- 172.21.15.92:0/2277478156 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe2d8131fa0 msgr2=0x7fe2d8095a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:54.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.447+0000 7fe2d659c640 1 --2- 172.21.15.92:0/2277478156 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe2d8131fa0 0x7fe2d8095a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:54.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.447+0000 7fe2d659c640 1 -- 172.21.15.92:0/2277478156 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe2d8128660 con 0x7fe2d8099200 2024-09-21T09:43:54.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.447+0000 7fe2d6d9d640 1 --2- 172.21.15.92:0/2277478156 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe2d81348d0 0x7fe2d8095f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:54.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.447+0000 7fe2d5d9b640 1 --2- 172.21.15.92:0/2277478156 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe2d8131fa0 0x7fe2d8095a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T09:43:54.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.447+0000 7fe2d659c640 1 --2- 172.21.15.92:0/2277478156 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe2d8099200 0x7fe2d80954e0 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7fe2c800cc50 tx=0x7fe2c800a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T09:43:54.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.447+0000 7fe2ceffd640 1 -- 172.21.15.92:0/2277478156 <== mon.0 v2:172.21.15.92:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe2c8016020 con 0x7fe2d8099200 2024-09-21T09:43:54.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.447+0000 7fe2ceffd640 1 -- 172.21.15.92:0/2277478156 <== mon.0 v2:172.21.15.92:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fe2c8012420 con 0x7fe2d8099200 2024-09-21T09:43:54.452 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.447+0000 7fe2dcfea640 1 -- 172.21.15.92:0/2277478156 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe2d80977f0 con 0x7fe2d8099200 2024-09-21T09:43:54.453 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.447+0000 7fe2ceffd640 1 -- 172.21.15.92:0/2277478156 <== mon.0 v2:172.21.15.92:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe2c800ce60 con 0x7fe2d8099200 2024-09-21T09:43:54.453 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.447+0000 7fe2dcfea640 1 -- 172.21.15.92:0/2277478156 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe2d80964a0 con 0x7fe2d8099200 2024-09-21T09:43:54.453 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.448+0000 7fe2dcfea640 1 -- 172.21.15.92:0/2277478156 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe2d80a1300 con 0x7fe2d8099200 2024-09-21T09:43:54.453 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.448+0000 7fe2ceffd640 1 -- 172.21.15.92:0/2277478156 <== mon.0 v2:172.21.15.92:3300/0 4 ==== mgrmap(e 9) ==== 63498+0+0 (secure 0 0 0) 0x7fe2c8019020 con 0x7fe2d8099200 2024-09-21T09:43:54.453 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.448+0000 7fe2ceffd640 1 -- 172.21.15.92:0/2277478156 <== mon.0 v2:172.21.15.92:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fe2c8069330 con 0x7fe2d8099200 2024-09-21T09:43:54.454 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.452+0000 7fe2ceffd640 1 -- 172.21.15.92:0/2277478156 <== mon.0 v2:172.21.15.92:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fe2d80a1300 con 0x7fe2d8099200 2024-09-21T09:43:54.603 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.600+0000 7fe2dcfea640 1 -- 172.21.15.92:0/2277478156 --> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] -- mon_command([{prefix=config set, name=mon_health_to_clog}] v 0) -- 0x7fe2d80699c0 con 0x7fe2d8099200 2024-09-21T09:43:54.608 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.605+0000 7fe2ceffd640 1 -- 172.21.15.92:0/2277478156 <== mon.0 v2:172.21.15.92:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_health_to_clog}]=0 v24)=0 v24) ==== 123+0+0 (secure 0 0 0) 0x7fe2c8020030 con 0x7fe2d8099200 2024-09-21T09:43:54.609 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.607+0000 7fe2ceffd640 1 -- 172.21.15.92:0/2277478156 <== mon.0 v2:172.21.15.92:3300/0 8 ==== config(6 keys) ==== 172+0+0 (secure 0 0 0) 0x7fe2c8045790 con 0x7fe2d8099200 2024-09-21T09:43:54.611 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.608+0000 7fe2dcfea640 1 -- 172.21.15.92:0/2277478156 >> v1:172.21.15.92:6800/2299163855 conn(0x7fe2b80514b0 legacy=0x7fe2b8053970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:54.611 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.608+0000 7fe2dcfea640 1 -- 172.21.15.92:0/2277478156 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe2d8099200 msgr2=0x7fe2d80954e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T09:43:54.611 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.608+0000 7fe2dcfea640 1 --2- 172.21.15.92:0/2277478156 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe2d8099200 0x7fe2d80954e0 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7fe2c800cc50 tx=0x7fe2c800a430 comp rx=0 tx=0).stop 2024-09-21T09:43:54.611 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.608+0000 7fe2dcfea640 1 -- 172.21.15.92:0/2277478156 shutdown_connections 2024-09-21T09:43:54.611 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.608+0000 7fe2dcfea640 1 --2- 172.21.15.92:0/2277478156 >> [v2:172.21.15.92:3302/0,v1:172.21.15.92:6791/0] conn(0x7fe2d81348d0 0x7fe2d8095f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:54.611 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.608+0000 7fe2dcfea640 1 --2- 172.21.15.92:0/2277478156 >> [v2:172.21.15.92:3301/0,v1:172.21.15.92:6790/0] conn(0x7fe2d8131fa0 0x7fe2d8095a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:54.611 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.608+0000 7fe2dcfea640 1 --2- 172.21.15.92:0/2277478156 >> [v2:172.21.15.92:3300/0,v1:172.21.15.92:6789/0] conn(0x7fe2d8099200 0x7fe2d80954e0 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T09:43:54.611 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.608+0000 7fe2dcfea640 1 -- 172.21.15.92:0/2277478156 >> 172.21.15.92:0/2277478156 conn(0x7fe2d80a1970 msgr2=0x7fe2d809cde0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T09:43:54.612 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.608+0000 7fe2dcfea640 1 -- 172.21.15.92:0/2277478156 shutdown_connections 2024-09-21T09:43:54.612 INFO:teuthology.orchestra.run.smithi092.stderr:2024-09-21T09:43:54.608+0000 7fe2dcfea640 1 -- 172.21.15.92:0/2277478156 wait complete. 2024-09-21T09:43:54.623 INFO:teuthology.misc:Shutting down mds daemons... 2024-09-21T09:43:54.623 INFO:teuthology.misc:Shutting down osd daemons... 2024-09-21T09:43:54.623 DEBUG:tasks.ceph.osd.0:waiting for process to exit 2024-09-21T09:43:54.623 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:43:54.655 INFO:tasks.ceph.osd.0:Stopped 2024-09-21T09:43:54.655 DEBUG:tasks.ceph.osd.1:waiting for process to exit 2024-09-21T09:43:54.655 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:43:54.723 INFO:tasks.ceph.osd.1:Stopped 2024-09-21T09:43:54.723 DEBUG:tasks.ceph.osd.2:waiting for process to exit 2024-09-21T09:43:54.723 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:43:54.794 INFO:tasks.ceph.osd.2:Stopped 2024-09-21T09:43:54.795 DEBUG:tasks.ceph.osd.3:waiting for process to exit 2024-09-21T09:43:54.795 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:43:54.869 INFO:tasks.ceph.osd.3:Stopped 2024-09-21T09:43:54.870 INFO:teuthology.misc:Shutting down mgr daemons... 2024-09-21T09:43:54.870 DEBUG:tasks.ceph.mgr.x:waiting for process to exit 2024-09-21T09:43:54.870 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:43:54.931 INFO:tasks.ceph.mgr.x:Stopped 2024-09-21T09:43:54.931 DEBUG:tasks.ceph.mgr.y:waiting for process to exit 2024-09-21T09:43:54.931 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:43:54.997 INFO:tasks.ceph.mgr.y:Stopped 2024-09-21T09:43:54.997 INFO:teuthology.misc:Shutting down mon daemons... 2024-09-21T09:43:54.997 DEBUG:tasks.ceph.mon.a:waiting for process to exit 2024-09-21T09:43:54.998 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:43:55.049 INFO:tasks.ceph.mon.a:Stopped 2024-09-21T09:43:55.049 DEBUG:tasks.ceph.mon.b:waiting for process to exit 2024-09-21T09:43:55.049 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:43:55.102 INFO:tasks.ceph.mon.b:Stopped 2024-09-21T09:43:55.103 DEBUG:tasks.ceph.mon.c:waiting for process to exit 2024-09-21T09:43:55.103 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T09:43:55.153 INFO:tasks.ceph.mon.c:Stopped 2024-09-21T09:43:55.153 INFO:tasks.ceph:Checking cluster log for badness... 2024-09-21T09:43:55.154 DEBUG:teuthology.orchestra.run.smithi092:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/ceph.log | egrep -v 'overall HEALTH_' | egrep -v '\(FS_' | egrep -v '\(MDS_' | egrep -v '\(OSD_' | egrep -v '\(MON_DOWN\)' | egrep -v '\(CACHE_POOL_' | egrep -v '\(POOL_' | egrep -v '\(MGR_DOWN\)' | egrep -v '\(PG_' | egrep -v '\(SMALLER_PGP_NUM\)' | egrep -v 'Monitor daemon marked osd' | egrep -v 'Behind on trimming' | egrep -v 'Manager daemon' | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v FS_DEGRADED | egrep -v 'fs.*is degraded' | egrep -v 'filesystem is degraded' | egrep -v FS_INLINE_DATA_DEPRECATED | egrep -v FS_WITH_FAILED_MDS | egrep -v MDS_ALL_DOWN | egrep -v 'filesystem is offline' | egrep -v MDS_DAMAGE | egrep -v MDS_DEGRADED | egrep -v MDS_FAILED | egrep -v MDS_INSUFFICIENT_STANDBY | egrep -v 'insufficient standby MDS daemons available' | egrep -v MDS_UP_LESS_THAN_MAX | egrep -v 'online, but wants' | egrep -v 'filesystem is online with fewer MDS than max_mds' | egrep -v POOL_APP_NOT_ENABLED | egrep -v 'do not have an application enabled' | egrep -v 'overall HEALTH_' | egrep -v 'Replacing daemon' | egrep -v 'deprecated feature inline_data' | egrep -v 'overall HEALTH_' | egrep -v '\(OSD_DOWN\)' | egrep -v '\(OSD_' | egrep -v 'but it is still running' | egrep -v 'is not responding' | egrep -v PG_AVAILABILITY | egrep -v PG_DEGRADED | egrep -v 'Reduced data availability' | egrep -v 'Degraded data redundancy' | egrep -v 'scrub mismatch' | egrep -v ScrubResult | egrep -v 'wrongly marked' | egrep -v '\(POOL_APP_NOT_ENABLED\)' | egrep -v '\(SLOW_OPS\)' | egrep -v 'overall HEALTH_' | egrep -v '\(MON_MSGR2_NOT_ENABLED\)' | egrep -v 'slow request' | head -n 1 2024-09-21T09:43:55.235 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-0 on ubuntu@smithi092.front.sepia.ceph.com 2024-09-21T09:43:55.235 DEBUG:teuthology.orchestra.run.smithi092:> sync && sudo umount -f /var/lib/ceph/osd/ceph-0 2024-09-21T09:43:55.410 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-1 on ubuntu@smithi092.front.sepia.ceph.com 2024-09-21T09:43:55.410 DEBUG:teuthology.orchestra.run.smithi092:> sync && sudo umount -f /var/lib/ceph/osd/ceph-1 2024-09-21T09:43:55.482 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-2 on ubuntu@smithi092.front.sepia.ceph.com 2024-09-21T09:43:55.483 DEBUG:teuthology.orchestra.run.smithi092:> sync && sudo umount -f /var/lib/ceph/osd/ceph-2 2024-09-21T09:43:55.550 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-3 on ubuntu@smithi092.front.sepia.ceph.com 2024-09-21T09:43:55.550 DEBUG:teuthology.orchestra.run.smithi092:> sync && sudo umount -f /var/lib/ceph/osd/ceph-3 2024-09-21T09:43:55.599 INFO:tasks.ceph:Archiving mon data... 2024-09-21T09:43:55.600 DEBUG:teuthology.misc:Transferring archived files from smithi092:/var/lib/ceph/mon/ceph-a to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913659/data/mon.a.tgz 2024-09-21T09:43:55.600 DEBUG:teuthology.orchestra.run.smithi092:> mktemp 2024-09-21T09:43:55.618 INFO:teuthology.orchestra.run.smithi092.stdout:/tmp/tmp.VJk5pUEC7A 2024-09-21T09:43:55.619 DEBUG:teuthology.orchestra.run.smithi092:> sudo tar cz -f - -C /var/lib/ceph/mon/ceph-a -- . > /tmp/tmp.VJk5pUEC7A 2024-09-21T09:43:55.778 DEBUG:teuthology.orchestra.run.smithi092:> sudo chmod 0666 /tmp/tmp.VJk5pUEC7A 2024-09-21T09:43:55.859 DEBUG:teuthology.orchestra.remote:smithi092:/tmp/tmp.VJk5pUEC7A is 231KB 2024-09-21T09:43:55.952 DEBUG:teuthology.orchestra.run.smithi092:> rm -fr /tmp/tmp.VJk5pUEC7A 2024-09-21T09:43:55.967 DEBUG:teuthology.misc:Transferring archived files from smithi092:/var/lib/ceph/mon/ceph-b to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913659/data/mon.b.tgz 2024-09-21T09:43:55.967 DEBUG:teuthology.orchestra.run.smithi092:> mktemp 2024-09-21T09:43:56.021 INFO:teuthology.orchestra.run.smithi092.stdout:/tmp/tmp.2mG5SjzPjJ 2024-09-21T09:43:56.021 DEBUG:teuthology.orchestra.run.smithi092:> sudo tar cz -f - -C /var/lib/ceph/mon/ceph-b -- . > /tmp/tmp.2mG5SjzPjJ 2024-09-21T09:43:56.136 DEBUG:teuthology.orchestra.run.smithi092:> sudo chmod 0666 /tmp/tmp.2mG5SjzPjJ 2024-09-21T09:43:56.218 DEBUG:teuthology.orchestra.remote:smithi092:/tmp/tmp.2mG5SjzPjJ is 278KB 2024-09-21T09:43:56.312 DEBUG:teuthology.orchestra.run.smithi092:> rm -fr /tmp/tmp.2mG5SjzPjJ 2024-09-21T09:43:56.326 DEBUG:teuthology.misc:Transferring archived files from smithi092:/var/lib/ceph/mon/ceph-c to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913659/data/mon.c.tgz 2024-09-21T09:43:56.327 DEBUG:teuthology.orchestra.run.smithi092:> mktemp 2024-09-21T09:43:56.381 INFO:teuthology.orchestra.run.smithi092.stdout:/tmp/tmp.kpjDjKEFVo 2024-09-21T09:43:56.381 DEBUG:teuthology.orchestra.run.smithi092:> sudo tar cz -f - -C /var/lib/ceph/mon/ceph-c -- . > /tmp/tmp.kpjDjKEFVo 2024-09-21T09:43:56.496 DEBUG:teuthology.orchestra.run.smithi092:> sudo chmod 0666 /tmp/tmp.kpjDjKEFVo 2024-09-21T09:43:56.578 DEBUG:teuthology.orchestra.remote:smithi092:/tmp/tmp.kpjDjKEFVo is 278KB 2024-09-21T09:43:56.672 DEBUG:teuthology.orchestra.run.smithi092:> rm -fr /tmp/tmp.kpjDjKEFVo 2024-09-21T09:43:56.687 INFO:tasks.ceph:Cleaning ceph cluster... 2024-09-21T09:43:56.687 DEBUG:teuthology.orchestra.run.smithi092:> sudo rm -rf -- /etc/ceph/ceph.conf /etc/ceph/ceph.keyring /home/ubuntu/cephtest/ceph.data /home/ubuntu/cephtest/ceph.monmap /home/ubuntu/cephtest/../*.pid 2024-09-21T09:43:56.807 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2024-09-21T09:43:56.807 INFO:tasks.ceph:Archiving crash dumps... 2024-09-21T09:43:56.808 DEBUG:teuthology.misc:Transferring archived files from smithi092:/var/lib/ceph/crash to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913659/remote/smithi092/crash 2024-09-21T09:43:56.809 DEBUG:teuthology.orchestra.run.smithi092:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2024-09-21T09:43:56.870 INFO:tasks.ceph:Compressing logs... 2024-09-21T09:43:56.870 DEBUG:teuthology.orchestra.run.smithi092:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T09:43:56.934 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-osd.0.log 2024-09-21T09:43:56.956 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33559.log 2024-09-21T09:43:56.956 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32611.log 2024-09-21T09:43:56.956 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-osd.0.log: /var/log/ceph/ceph-client.admin.33559.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40065.log 2024-09-21T09:43:56.956 INFO:teuthology.orchestra.run.smithi092.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.33559.log.gz 2024-09-21T09:43:56.956 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.32611.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33439.log 2024-09-21T09:43:56.957 INFO:teuthology.orchestra.run.smithi092.stderr: 77.8% -- replaced with /var/log/ceph/ceph-client.admin.32611.log.gz 2024-09-21T09:43:56.957 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32678.log 2024-09-21T09:43:56.957 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.40065.log: /var/log/ceph/ceph-client.admin.33439.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47017.log 2024-09-21T09:43:56.957 INFO:teuthology.orchestra.run.smithi092.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.40065.log.gz 2024-09-21T09:43:56.957 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose/var/log/ceph/ceph-client.admin.32678.log: -- /var/log/ceph/ceph-client.admin.47144.log 2024-09-21T09:43:56.957 INFO:teuthology.orchestra.run.smithi092.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.33439.log.gz 2024-09-21T09:43:56.957 INFO:teuthology.orchestra.run.smithi092.stderr: 87.5% -- replaced with /var/log/ceph/ceph-client.admin.32678.log.gz 2024-09-21T09:43:56.957 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.47017.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40066.log 2024-09-21T09:43:56.957 INFO:teuthology.orchestra.run.smithi092.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.47017.log.gz 2024-09-21T09:43:56.957 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.47144.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41692.log 2024-09-21T09:43:56.957 INFO:teuthology.orchestra.run.smithi092.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.47144.log.gz 2024-09-21T09:43:56.957 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.40066.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41694.log 2024-09-21T09:43:56.957 INFO:teuthology.orchestra.run.smithi092.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.40066.log.gz 2024-09-21T09:43:56.957 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.41692.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32845.log 2024-09-21T09:43:56.958 INFO:teuthology.orchestra.run.smithi092.stderr: 85.8% -- replaced with /var/log/ceph/ceph-client.admin.41692.log.gz 2024-09-21T09:43:56.958 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.41694.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39728.log 2024-09-21T09:43:56.958 INFO:teuthology.orchestra.run.smithi092.stderr: 85.5% -- replaced with /var/log/ceph/ceph-client.admin.41694.log.gz 2024-09-21T09:43:56.958 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.32845.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46621.log 2024-09-21T09:43:56.958 INFO:teuthology.orchestra.run.smithi092.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.32845.log.gz 2024-09-21T09:43:56.958 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.39728.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39878.log 2024-09-21T09:43:56.958 INFO:teuthology.orchestra.run.smithi092.stderr: 85.6% -- replaced with /var/log/ceph/ceph-client.admin.39728.log.gz 2024-09-21T09:43:56.958 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.46621.log: gzip -5 --verbose -- /var/log/ceph/ceph-osd.1.log 2024-09-21T09:43:56.958 INFO:teuthology.orchestra.run.smithi092.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.46621.log.gz 2024-09-21T09:43:56.958 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.39878.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42774.log 2024-09-21T09:43:56.958 INFO:teuthology.orchestra.run.smithi092.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.39878.log.gz 2024-09-21T09:43:56.958 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41819.log 2024-09-21T09:43:56.958 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.42774.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47377.log 2024-09-21T09:43:56.958 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.41819.log: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.42774.log.gz 2024-09-21T09:43:56.959 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39629.log 2024-09-21T09:43:56.959 INFO:teuthology.orchestra.run.smithi092.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.41819.log.gz 2024-09-21T09:43:56.959 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.47377.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46414.log 2024-09-21T09:43:56.959 INFO:teuthology.orchestra.run.smithi092.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.47377.log.gz 2024-09-21T09:43:56.959 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.39629.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47839.log 2024-09-21T09:43:56.959 INFO:teuthology.orchestra.run.smithi092.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.39629.log.gz 2024-09-21T09:43:56.959 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32705.log 2024-09-21T09:43:56.959 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.46414.log: /var/log/ceph/ceph-client.admin.47839.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42941.log 2024-09-21T09:43:56.959 INFO:teuthology.orchestra.run.smithi092.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.46414.log.gz 2024-09-21T09:43:56.959 INFO:teuthology.orchestra.run.smithi092.stderr: 85.6% -- replaced with /var/log/ceph/ceph-client.admin.47839.log.gz 2024-09-21T09:43:56.959 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.32705.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47145.log 2024-09-21T09:43:56.959 INFO:teuthology.orchestra.run.smithi092.stderr: 0.0% -- replaced with /var/log/ceph/ceph-client.admin.32705.log.gz 2024-09-21T09:43:56.959 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.42941.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48005.log 2024-09-21T09:43:56.959 INFO:teuthology.orchestra.run.smithi092.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.42941.log.gz 2024-09-21T09:43:56.959 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.47145.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41971.log 2024-09-21T09:43:56.960 INFO:teuthology.orchestra.run.smithi092.stderr: 87.1%gzip -- replaced with /var/log/ceph/ceph-client.admin.47145.log.gz 2024-09-21T09:43:56.960 INFO:teuthology.orchestra.run.smithi092.stderr: -5 --verbose -- /var/log/ceph/ceph-client.admin.46454.log 2024-09-21T09:43:56.960 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.48005.log: /var/log/ceph/ceph-client.admin.41971.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32901.log 2024-09-21T09:43:56.960 INFO:teuthology.orchestra.run.smithi092.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.48005.log.gz 2024-09-21T09:43:56.960 INFO:teuthology.orchestra.run.smithi092.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.41971.log.gz 2024-09-21T09:43:56.960 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40007.log 2024-09-21T09:43:56.960 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.46454.log: /var/log/ceph/ceph-client.admin.32901.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47293.log 2024-09-21T09:43:56.960 INFO:teuthology.orchestra.run.smithi092.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.46454.log.gz 2024-09-21T09:43:56.960 INFO:teuthology.orchestra.run.smithi092.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.32901.log.gz 2024-09-21T09:43:56.960 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.40007.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32985.log 2024-09-21T09:43:56.960 INFO:teuthology.orchestra.run.smithi092.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.40007.log.gz 2024-09-21T09:43:56.960 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40067.log 2024-09-21T09:43:56.960 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.47293.log: gzip -5 --verbose --/var/log/ceph/ceph-client.admin.32985.log: /var/log/ceph/ceph-client.admin.46598.log 2024-09-21T09:43:56.960 INFO:teuthology.orchestra.run.smithi092.stderr: 85.9% -- replaced with /var/log/ceph/ceph-client.admin.47293.log.gz 2024-09-21T09:43:56.961 INFO:teuthology.orchestra.run.smithi092.stderr: 0.0% -- replaced with /var/log/ceph/ceph-client.admin.32985.log.gz 2024-09-21T09:43:56.961 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.40067.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32957.log 2024-09-21T09:43:56.961 INFO:teuthology.orchestra.run.smithi092.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.40067.log.gz 2024-09-21T09:43:56.961 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.46598.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42299.log 2024-09-21T09:43:56.961 INFO:teuthology.orchestra.run.smithi092.stderr: 86.4% -- replaced with /var/log/ceph/ceph-client.admin.46598.log.gz 2024-09-21T09:43:56.961 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41551.log 2024-09-21T09:43:56.961 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.32957.log: /var/log/ceph/ceph-client.admin.42299.log: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.32957.log.gz 2024-09-21T09:43:56.961 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40175.log 2024-09-21T09:43:56.961 INFO:teuthology.orchestra.run.smithi092.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.42299.log.gz 2024-09-21T09:43:56.961 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42578.log 2024-09-21T09:43:56.961 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.41551.log: /var/log/ceph/ceph-client.admin.40175.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47016.log 2024-09-21T09:43:56.961 INFO:teuthology.orchestra.run.smithi092.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.41551.log.gz 2024-09-21T09:43:56.961 INFO:teuthology.orchestra.run.smithi092.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.40175.log.gz 2024-09-21T09:43:56.961 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.42578.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41693.log 2024-09-21T09:43:56.962 INFO:teuthology.orchestra.run.smithi092.stderr: 86.0% -- replaced with /var/log/ceph/ceph-client.admin.42578.log.gz 2024-09-21T09:43:56.962 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.47016.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39545.log 2024-09-21T09:43:56.962 INFO:teuthology.orchestra.run.smithi092.stderr: 86.2% -- replaced with /var/log/ceph/ceph-client.admin.47016.log.gz 2024-09-21T09:43:56.962 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41843.log 2024-09-21T09:43:56.962 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.41693.log: /var/log/ceph/ceph-client.admin.39545.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46374.log 2024-09-21T09:43:56.962 INFO:teuthology.orchestra.run.smithi092.stderr: 85.9% -- replaced with /var/log/ceph/ceph-client.admin.41693.log.gz 2024-09-21T09:43:56.962 INFO:teuthology.orchestra.run.smithi092.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.39545.log.gz 2024-09-21T09:43:56.962 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.41843.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32440.log 2024-09-21T09:43:56.962 INFO:teuthology.orchestra.run.smithi092.stderr:gzip 86.9% -5 -- replaced with /var/log/ceph/ceph-client.admin.41843.log.gz --verbose 2024-09-21T09:43:56.962 INFO:teuthology.orchestra.run.smithi092.stderr: -- /var/log/ceph/ceph-client.admin.39729.log 2024-09-21T09:43:56.962 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.46374.log: /var/log/ceph/ceph-client.admin.32440.log: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.46374.log.gz 2024-09-21T09:43:56.962 INFO:teuthology.orchestra.run.smithi092.stderr: 0.0% -- replaced with /var/log/ceph/ceph-client.admin.32440.log.gz 2024-09-21T09:43:56.962 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40427.log 2024-09-21T09:43:56.962 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.audit.log 2024-09-21T09:43:56.963 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.39729.log: /var/log/ceph/ceph-client.admin.40427.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42071.log 2024-09-21T09:43:56.963 INFO:teuthology.orchestra.run.smithi092.stderr: 85.8% -- replaced with /var/log/ceph/ceph-client.admin.39729.log.gz 2024-09-21T09:43:56.963 INFO:teuthology.orchestra.run.smithi092.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.40427.log.gz 2024-09-21T09:43:56.963 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33677.log 2024-09-21T09:43:56.963 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.37415.log 2024-09-21T09:43:56.963 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.42071.log: /var/log/ceph/ceph-client.admin.33677.log: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.42071.log.gz 2024-09-21T09:43:56.963 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46747.log 2024-09-21T09:43:56.963 INFO:teuthology.orchestra.run.smithi092.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.33677.log.gz 2024-09-21T09:43:56.963 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.37415.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32733.log 2024-09-21T09:43:56.963 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.46747.log: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.37415.log.gz 2024-09-21T09:43:56.963 INFO:teuthology.orchestra.run.smithi092.stderr: 94.9% -- replaced with /var/log/ceph/ceph.audit.log.gz 2024-09-21T09:43:56.963 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42230.log 2024-09-21T09:43:56.963 INFO:teuthology.orchestra.run.smithi092.stderr: 86.3% -- replaced with /var/log/ceph/ceph-client.admin.46747.log.gz 2024-09-21T09:43:56.963 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39899.log 2024-09-21T09:43:56.964 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.32733.log: /var/log/ceph/ceph-client.admin.42230.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47015.log 2024-09-21T09:43:56.964 INFO:teuthology.orchestra.run.smithi092.stderr: 87.3% -- replaced with /var/log/ceph/ceph-client.admin.32733.log.gz 2024-09-21T09:43:56.964 INFO:teuthology.orchestra.run.smithi092.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.42230.log.gz 2024-09-21T09:43:56.964 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.39899.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47629.log 2024-09-21T09:43:56.964 INFO:teuthology.orchestra.run.smithi092.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.39899.log.gz 2024-09-21T09:43:56.964 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32639.log 2024-09-21T09:43:56.964 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.47015.log: /var/log/ceph/ceph-client.admin.47629.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32817.log 2024-09-21T09:43:56.964 INFO:teuthology.orchestra.run.smithi092.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.47015.log.gz 2024-09-21T09:43:56.964 INFO:teuthology.orchestra.run.smithi092.stderr: 86.4% -- replaced with /var/log/ceph/ceph-client.admin.47629.log.gz 2024-09-21T09:43:56.964 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.32639.log: 0.0% -- replaced with /var/log/ceph/ceph-client.admin.32639.log.gz 2024-09-21T09:43:56.964 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33519.log 2024-09-21T09:43:56.964 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46494.log 2024-09-21T09:43:56.964 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.32817.log: /var/log/ceph/ceph-client.admin.33519.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40385.log 2024-09-21T09:43:56.964 INFO:teuthology.orchestra.run.smithi092.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.32817.log.gz 2024-09-21T09:43:56.964 INFO:teuthology.orchestra.run.smithi092.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.33519.log.gz 2024-09-21T09:43:56.965 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.46494.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33479.log 2024-09-21T09:43:56.965 INFO:teuthology.orchestra.run.smithi092.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.46494.log.gz 2024-09-21T09:43:56.965 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.40385.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39359.log 2024-09-21T09:43:56.965 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.33479.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32789.log 2024-09-21T09:43:56.965 INFO:teuthology.orchestra.run.smithi092.stderr: 86.0% -- replaced with /var/log/ceph/ceph-client.admin.40385.log.gz 2024-09-21T09:43:56.965 INFO:teuthology.orchestra.run.smithi092.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.33479.log.gz 2024-09-21T09:43:56.965 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.39359.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47755.log 2024-09-21T09:43:56.965 INFO:teuthology.orchestra.run.smithi092.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.39359.log.gz 2024-09-21T09:43:56.965 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41831.log 2024-09-21T09:43:56.965 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.32789.log: /var/log/ceph/ceph-client.admin.47755.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33327.log 2024-09-21T09:43:56.965 INFO:teuthology.orchestra.run.smithi092.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.32789.log.gz 2024-09-21T09:43:56.965 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.41831.log: 86.0% -- replaced with /var/log/ceph/ceph-client.admin.47755.log.gz 2024-09-21T09:43:56.965 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40233.log 2024-09-21T09:43:56.965 INFO:teuthology.orchestra.run.smithi092.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.41831.log.gz 2024-09-21T09:43:56.966 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.33327.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39730.log 2024-09-21T09:43:56.966 INFO:teuthology.orchestra.run.smithi092.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.33327.log.gz 2024-09-21T09:43:56.966 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47013.log 2024-09-21T09:43:56.966 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.40233.log: /var/log/ceph/ceph-client.admin.39730.log: gzip -5 --verbose -- /var/log/ceph/ceph-mon.c.log 2024-09-21T09:43:56.966 INFO:teuthology.orchestra.run.smithi092.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.40233.log.gz 2024-09-21T09:43:56.966 INFO:teuthology.orchestra.run.smithi092.stderr: 85.8% -- replaced with /var/log/ceph/ceph-client.admin.39730.log.gz 2024-09-21T09:43:56.966 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.47013.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42490.log 2024-09-21T09:43:56.966 INFO:teuthology.orchestra.run.smithi092.stderr: 86.1% -- replaced with /var/log/ceph/ceph-client.admin.47013.log.gz 2024-09-21T09:43:56.966 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40234.log 2024-09-21T09:43:56.966 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-mon.c.log: /var/log/ceph/ceph-client.admin.42490.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46846.log 2024-09-21T09:43:56.966 INFO:teuthology.orchestra.run.smithi092.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.42490.log.gz 2024-09-21T09:43:56.966 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.40234.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46534.log 2024-09-21T09:43:56.966 INFO:teuthology.orchestra.run.smithi092.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.40234.log.gz 2024-09-21T09:43:56.966 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.46846.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42478.log 2024-09-21T09:43:56.990 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.46534.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41593.log 2024-09-21T09:43:56.990 INFO:teuthology.orchestra.run.smithi092.stderr: 86.3% -- replaced with /var/log/ceph/ceph-client.admin.46846.log.gz 2024-09-21T09:43:56.990 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.42478.log: 87.3% -- replaced with /var/log/ceph/ceph-client.admin.46534.log.gz 2024-09-21T09:43:56.990 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33013.log 2024-09-21T09:43:56.990 INFO:teuthology.orchestra.run.smithi092.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.42478.log.gz 2024-09-21T09:43:56.990 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.41593.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47503.log 2024-09-21T09:43:56.990 INFO:teuthology.orchestra.run.smithi092.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.41593.log.gz 2024-09-21T09:43:56.990 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47461.log 2024-09-21T09:43:56.990 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.33013.log: 0.0% -- replaced with /var/log/ceph/ceph-client.admin.33013.log.gz 2024-09-21T09:43:56.990 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.47503.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39877.log 2024-09-21T09:43:56.990 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.47461.log: 85.9% -- replaced with /var/log/ceph/ceph-client.admin.47503.log.gz 2024-09-21T09:43:56.990 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32761.log 2024-09-21T09:43:56.991 INFO:teuthology.orchestra.run.smithi092.stderr: 86.4% -- replaced with /var/log/ceph/ceph-client.admin.47461.log.gz 2024-09-21T09:43:56.991 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33041.log 2024-09-21T09:43:56.991 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.39877.log: /var/log/ceph/ceph-client.admin.32761.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39587.log 2024-09-21T09:43:56.991 INFO:teuthology.orchestra.run.smithi092.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.39877.log.gz 2024-09-21T09:43:56.991 INFO:teuthology.orchestra.run.smithi092.stderr: 0.0% -- replaced with /var/log/ceph/ceph-client.admin.32761.log.gz 2024-09-21T09:43:56.991 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.33041.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46663.log 2024-09-21T09:43:56.991 INFO:teuthology.orchestra.run.smithi092.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.33041.log.gz 2024-09-21T09:43:56.991 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.39587.log: gzip -5 --verbose -- /var/log/ceph/ceph-mon.a.log 2024-09-21T09:43:56.991 INFO:teuthology.orchestra.run.smithi092.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.39587.log.gz 2024-09-21T09:43:56.991 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46848.log 2024-09-21T09:43:56.991 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.46663.log: /var/log/ceph/ceph-mon.a.log: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.46663.log.gz 2024-09-21T09:43:56.991 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47419.log 2024-09-21T09:43:56.991 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47185.log 2024-09-21T09:43:56.991 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.46848.log: /var/log/ceph/ceph-client.admin.47419.log: 85.5% -- replaced with /var/log/ceph/ceph-client.admin.46848.log.gz 2024-09-21T09:43:56.991 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47965.log 2024-09-21T09:43:56.992 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.47185.log: 86.3% -- replaced with /var/log/ceph/ceph-client.admin.47419.log.gz 2024-09-21T09:43:56.992 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47587.log 2024-09-21T09:43:56.992 INFO:teuthology.orchestra.run.smithi092.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.47185.log.gz 2024-09-21T09:43:56.992 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.47965.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46332.log 2024-09-21T09:43:56.992 INFO:teuthology.orchestra.run.smithi092.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.47965.log.gz 2024-09-21T09:43:56.992 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.47587.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42704.log 2024-09-21T09:43:56.992 INFO:teuthology.orchestra.run.smithi092.stderr: 85.6% -- replaced with /var/log/ceph/ceph-client.admin.47587.log.gz 2024-09-21T09:43:56.992 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.46332.log: gzip -5 --verbose -- /var/log/ceph/ceph-osd.2.log 2024-09-21T09:43:56.992 INFO:teuthology.orchestra.run.smithi092.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.46332.log.gz 2024-09-21T09:43:56.992 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.42704.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42620.log 2024-09-21T09:43:56.992 INFO:teuthology.orchestra.run.smithi092.stderr: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.42704.log.gz 2024-09-21T09:43:56.992 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42250.log 2024-09-21T09:43:56.992 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.42620.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39451.log 2024-09-21T09:43:56.992 INFO:teuthology.orchestra.run.smithi092.stderr: 86.2% -- replaced with /var/log/ceph/ceph-client.admin.42620.log.gz 2024-09-21T09:43:56.993 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.42250.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46847.log 2024-09-21T09:43:56.993 INFO:teuthology.orchestra.run.smithi092.stderr: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.42250.log.gz 2024-09-21T09:43:56.993 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.39451.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41468.log 2024-09-21T09:43:56.993 INFO:teuthology.orchestra.run.smithi092.stderr: 85.3% -- replaced with /var/log/ceph/ceph-client.admin.39451.log.gz 2024-09-21T09:43:56.993 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47797.log 2024-09-21T09:43:56.993 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.41468.log: /var/log/ceph/ceph-client.admin.46847.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46705.log 2024-09-21T09:43:56.993 INFO:teuthology.orchestra.run.smithi092.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.41468.log.gz 2024-09-21T09:43:56.993 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.47797.log: 86.0% -- replaced with /var/log/ceph/ceph-client.admin.46847.log.gz 2024-09-21T09:43:56.993 INFO:teuthology.orchestra.run.smithi092.stderr: 85.9% -- replaced with /var/log/ceph/ceph-client.admin.47797.log.gz 2024-09-21T09:43:56.993 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-mgr.y.log 2024-09-21T09:43:56.993 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42011.log 2024-09-21T09:43:56.993 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.46705.log: /var/log/ceph/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.43021.log 2024-09-21T09:43:56.993 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.42011.log: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.46705.log.gz 2024-09-21T09:43:56.993 INFO:teuthology.orchestra.run.smithi092.stderr: 86.7%gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.26543.log 2024-09-21T09:43:56.994 INFO:teuthology.orchestra.run.smithi092.stderr: -- replaced with /var/log/ceph/ceph-client.admin.42011.log.gz 2024-09-21T09:43:56.994 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.43021.log: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.43021.log.gz 2024-09-21T09:43:56.994 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40507.log 2024-09-21T09:43:56.994 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.log 2024-09-21T09:43:56.994 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.40507.log: /var/log/ceph/ceph.tmp-client.admin.26543.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.26543.log.gz 2024-09-21T09:43:56.994 INFO:teuthology.orchestra.run.smithi092.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.40507.log.gz 2024-09-21T09:43:56.994 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42031.log 2024-09-21T09:43:56.994 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46849.log 2024-09-21T09:43:56.994 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.42031.log: /var/log/ceph/ceph.log: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.42031.log.gz 2024-09-21T09:43:56.994 INFO:teuthology.orchestra.run.smithi092.stderr: 93.7% -- replaced with /var/log/ceph/ceph.log.gz 2024-09-21T09:43:56.994 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32873.log 2024-09-21T09:43:56.994 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.46849.log: 86.0% -- replaced with /var/log/ceph/ceph-client.admin.46849.log.gz 2024-09-21T09:43:56.994 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.44507.log 2024-09-21T09:43:56.994 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-mgr.x.log 2024-09-21T09:43:56.995 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.44507.log: /var/log/ceph/ceph-client.admin.32873.log: 86.5% 87.0% -- replaced with /var/log/ceph/ceph-client.admin.32873.log.gz 2024-09-21T09:43:56.995 INFO:teuthology.orchestra.run.smithi092.stderr: -- replaced with /var/log/ceph/ceph-client.admin.44507.log.gz 2024-09-21T09:43:56.995 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42856.log 2024-09-21T09:43:56.995 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47545.log 2024-09-21T09:43:56.995 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.42856.log: /var/log/ceph/ceph-mgr.x.log: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.42856.log.gz 2024-09-21T09:43:56.995 INFO:teuthology.orchestra.run.smithi092.stderr: 91.8% -- replaced with /var/log/ceph/ceph-mgr.x.log.gz 2024-09-21T09:43:56.995 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32462.log 2024-09-21T09:43:56.995 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42304.log 2024-09-21T09:43:56.995 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.32462.log: /var/log/ceph/ceph-client.admin.47545.log: 0.0% -- replaced with /var/log/ceph/ceph-client.admin.32462.log.gz 2024-09-21T09:43:56.995 INFO:teuthology.orchestra.run.smithi092.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.47545.log.gz 2024-09-21T09:43:56.995 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47881.log 2024-09-21T09:43:56.995 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39303.log 2024-09-21T09:43:56.995 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.47881.log: /var/log/ceph/ceph-client.admin.42304.log: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.42304.log.gz 2024-09-21T09:43:56.995 INFO:teuthology.orchestra.run.smithi092.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.47881.log.gz 2024-09-21T09:43:56.996 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39879.log 2024-09-21T09:43:56.996 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41883.log 2024-09-21T09:43:56.996 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.39879.log: /var/log/ceph/ceph-client.admin.39303.log: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.39303.log.gz 2024-09-21T09:43:56.996 INFO:teuthology.orchestra.run.smithi092.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.39879.log.gz 2024-09-21T09:43:56.996 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-mon.b.log 2024-09-21T09:43:56.996 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42410.log 2024-09-21T09:43:56.996 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-mon.b.log: /var/log/ceph/ceph-client.admin.41883.log: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.41883.log.gz 2024-09-21T09:43:56.996 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42430.log 2024-09-21T09:43:56.996 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.42410.log: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.42410.log.gz 2024-09-21T09:43:56.996 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33840.log 2024-09-21T09:43:56.996 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.42430.log: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.42430.log.gz 2024-09-21T09:43:56.996 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47671.log 2024-09-21T09:43:56.996 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.33840.log: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.33840.log.gz 2024-09-21T09:43:56.996 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40343.log 2024-09-21T09:43:56.997 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.47671.log: 85.5% -- replaced with /var/log/ceph/ceph-client.admin.47671.log.gz 2024-09-21T09:43:57.006 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42662.log 2024-09-21T09:43:57.006 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.40343.log: 85.9% -- replaced with /var/log/ceph/ceph-client.admin.40343.log.gz 2024-09-21T09:43:57.006 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39731.log 2024-09-21T09:43:57.006 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.42662.log: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.42662.log.gz 2024-09-21T09:43:57.006 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47335.log 2024-09-21T09:43:57.006 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.39731.log: 85.8% -- replaced with /var/log/ceph/ceph-client.admin.39731.log.gz 2024-09-21T09:43:57.006 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40467.log 2024-09-21T09:43:57.006 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.47335.log: 85.8% -- replaced with /var/log/ceph/ceph-client.admin.47335.log.gz 93.6% -- replaced with /var/log/ceph/ceph-osd.0.log.gz 2024-09-21T09:43:57.006 INFO:teuthology.orchestra.run.smithi092.stderr: 2024-09-21T09:43:57.006 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39503.log 2024-09-21T09:43:57.006 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.40467.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47713.log 2024-09-21T09:43:57.007 INFO:teuthology.orchestra.run.smithi092.stderr: 86.4% -- replaced with /var/log/ceph/ceph-client.admin.40467.log.gz 2024-09-21T09:43:57.007 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.39503.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46575.log 2024-09-21T09:43:57.007 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.47713.log: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.39503.log.gz 2024-09-21T09:43:57.007 INFO:teuthology.orchestra.run.smithi092.stderr: 86.4%gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47923.log 2024-09-21T09:43:57.007 INFO:teuthology.orchestra.run.smithi092.stderr: -- replaced with /var/log/ceph/ceph-client.admin.47713.log.gz 2024-09-21T09:43:57.007 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-osd.3.log 2024-09-21T09:43:57.007 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.46575.log: /var/log/ceph/ceph-client.admin.47923.log: 87.3% -- replaced with /var/log/ceph/ceph-client.admin.46575.log.gz 2024-09-21T09:43:57.007 INFO:teuthology.orchestra.run.smithi092.stderr: 85.9% -- replaced with /var/log/ceph/ceph-client.admin.47923.log.gzgzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47184.log 2024-09-21T09:43:57.007 INFO:teuthology.orchestra.run.smithi092.stderr: 2024-09-21T09:43:57.007 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32929.log 2024-09-21T09:43:57.007 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-osd.3.log: /var/log/ceph/ceph-client.admin.47184.log: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.47184.log.gz 2024-09-21T09:43:57.007 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40235.log 2024-09-21T09:43:57.007 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.32929.log: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.32929.log.gz 2024-09-21T09:43:57.007 INFO:teuthology.orchestra.run.smithi092.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41695.log 2024-09-21T09:43:57.008 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.40235.log: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.40235.log.gz 2024-09-21T09:43:57.008 INFO:teuthology.orchestra.run.smithi092.stderr:/var/log/ceph/ceph-client.admin.41695.log: 85.3% -- replaced with /var/log/ceph/ceph-client.admin.41695.log.gz 2024-09-21T09:43:57.008 INFO:teuthology.orchestra.run.smithi092.stderr: 93.2% -- replaced with /var/log/ceph/ceph-osd.2.log.gz 2024-09-21T09:43:57.008 INFO:teuthology.orchestra.run.smithi092.stderr: 90.6% -- replaced with /var/log/ceph/ceph-mgr.y.log.gz 2024-09-21T09:43:57.027 INFO:teuthology.orchestra.run.smithi092.stderr: 93.8% -- replaced with /var/log/ceph/ceph-osd.1.log.gz 2024-09-21T09:43:57.029 INFO:teuthology.orchestra.run.smithi092.stderr: 91.8% -- replaced with /var/log/ceph/ceph-mon.c.log.gz 2024-09-21T09:43:57.047 INFO:teuthology.orchestra.run.smithi092.stderr: 91.5% -- replaced with /var/log/ceph/ceph-mon.b.log.gz 2024-09-21T09:43:57.070 INFO:teuthology.orchestra.run.smithi092.stderr: 93.3% -- replaced with /var/log/ceph/ceph-osd.3.log.gz 2024-09-21T09:43:57.098 INFO:teuthology.orchestra.run.smithi092.stderr: 90.9% -- replaced with /var/log/ceph/ceph-mon.a.log.gz 2024-09-21T09:43:57.100 INFO:teuthology.orchestra.run.smithi092.stderr: 2024-09-21T09:43:57.100 INFO:teuthology.orchestra.run.smithi092.stderr:real 0m0.175s 2024-09-21T09:43:57.101 INFO:teuthology.orchestra.run.smithi092.stderr:user 0m0.626s 2024-09-21T09:43:57.101 INFO:teuthology.orchestra.run.smithi092.stderr:sys 0m0.109s 2024-09-21T09:43:57.101 INFO:tasks.ceph:Archiving logs... 2024-09-21T09:43:57.101 DEBUG:teuthology.misc:Transferring archived files from smithi092:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913659/remote/smithi092/log 2024-09-21T09:43:57.102 DEBUG:teuthology.orchestra.run.smithi092:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T09:43:57.336 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-21T09:43:57.353 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-21T09:43:57.353 DEBUG:teuthology.orchestra.run.smithi092:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-21T09:43:57.400 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-21T09:43:57.425 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-21T09:43:57.425 DEBUG:teuthology.orchestra.run.smithi092:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T09:43:57.439 INFO:teuthology.orchestra.run.smithi092.stderr:bash: line 1: ntpq: command not found 2024-09-21T09:43:57.443 INFO:teuthology.orchestra.run.smithi092.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T09:43:57.444 INFO:teuthology.orchestra.run.smithi092.stdout:=============================================================================== 2024-09-21T09:43:57.444 INFO:teuthology.orchestra.run.smithi092.stdout:^+ hv01.front.sepia.ceph.com 3 6 377 1 +12us[ +13us] +/- 35ms 2024-09-21T09:43:57.444 INFO:teuthology.orchestra.run.smithi092.stdout:^* hv02.front.sepia.ceph.com 3 6 377 0 +252us[ +253us] +/- 59ms 2024-09-21T09:43:57.444 INFO:teuthology.orchestra.run.smithi092.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 1 -366us[ -365us] +/- 80ms 2024-09-21T09:43:57.444 INFO:teuthology.orchestra.run.smithi092.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:43:57.444 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-21T09:43:57.483 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-21T09:43:57.484 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-21T09:43:57.507 DEBUG:teuthology.orchestra.run.smithi092:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T09:43:57.571 DEBUG:teuthology.orchestra.run.smithi092:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T09:43:57.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:43:57.598 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-21T09:43:57.609 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-21T09:43:57.634 INFO:teuthology.task.internal:Duration was 488.609842 seconds 2024-09-21T09:43:57.635 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-21T09:43:57.668 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-21T09:43:57.668 DEBUG:teuthology.orchestra.run.smithi092:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T09:43:57.706 INFO:teuthology.orchestra.run.smithi092.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T09:43:58.043 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-21T09:43:58.044 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi092.front.sepia.ceph.com 2024-09-21T09:43:58.044 DEBUG:teuthology.orchestra.run.smithi092:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T09:43:58.104 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-21T09:43:58.104 DEBUG:teuthology.orchestra.run.smithi092:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T09:43:58.169 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-21T09:43:58.170 DEBUG:teuthology.orchestra.run.smithi092:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T09:43:58.312 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-21T09:43:58.323 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-21T09:43:58.324 DEBUG:teuthology.orchestra.run.smithi092:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T09:43:58.348 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-21T09:43:58.360 DEBUG:teuthology.orchestra.run.smithi092:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T09:43:58.412 INFO:teuthology.orchestra.run.smithi092.stdout:kernel.core_pattern = core 2024-09-21T09:43:58.485 DEBUG:teuthology.orchestra.run.smithi092:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T09:43:58.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:43:58.541 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-21T09:43:58.552 INFO:teuthology.task.internal:Transferring archived files... 2024-09-21T09:43:58.553 DEBUG:teuthology.misc:Transferring archived files from smithi092:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913659/remote/smithi092 2024-09-21T09:43:58.553 DEBUG:teuthology.orchestra.run.smithi092:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T09:43:58.613 INFO:teuthology.task.internal:Removing archive directory... 2024-09-21T09:43:58.613 DEBUG:teuthology.orchestra.run.smithi092:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T09:43:58.664 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-21T09:43:58.676 INFO:teuthology.task.internal:Not uploading archives. 2024-09-21T09:43:58.676 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-21T09:43:58.702 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-21T09:43:58.702 DEBUG:teuthology.orchestra.run.smithi092:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T09:43:58.718 INFO:teuthology.orchestra.run.smithi092.stdout: 265813 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 09:43 /home/ubuntu/cephtest 2024-09-21T09:43:58.719 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-21T09:43:58.736 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-21T09:43:58.807 INFO:teuthology.run:Summary data: description: upgrade/cephfs/nofs/{bluestore-bitmap centos_9.stream conf/{client mds mgr mon osd} no-mds-cluster overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn pg_health} tasks/{0-from/squid 1-upgrade}} duration: 488.6098418235779 flavor: default owner: scheduled_teuthology@teuthology success: true 2024-09-21T09:43:58.807 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T09:43:58.877 INFO:teuthology.run:pass