2024-09-21T01:35:50.202 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-21T01:35:50.227 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T01:35:50.318 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/pdonnell-2024-09-20_23:48:22-fs-wip-pdonnell-testing-20240920.212106-debug-distro-default-smithi/7913395 branch: wip-pdonnell-testing-20240920.212106-debug description: fs/upgrade/nofs/{bluestore-bitmap centos_9.stream conf/{client mds mgr mon osd} no-mds-cluster overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn pg_health} tasks/{0-from/reef 1-upgrade}} email: pdonnell@redhat.com first_in_suite: false job_id: '7913395' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'install ceph/reef latest ' name: pdonnell-2024-09-20_23:48:22-fs-wip-pdonnell-testing-20240920.212106-debug-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-pdonnell-testing-20240920.212106-debug ceph: cluster-conf: client: client mount timeout: 600 debug client: 20 debug ms: 1 rados mon op timeout: 900 rados osd op timeout: 900 mds: debug mds: 20 debug mds balancer: 20 debug ms: 2 mds debug frag: true mds debug scatterstat: true mds op complaint time: 180 mds verify scatter: true osd op complaint time: 180 rados mon op timeout: 900 rados osd op timeout: 900 mgr: client mount timeout: 30 debug client: 20 debug mgr: 20 debug ms: 1 mon warn on pool no app: false mon: mon op complaint time: 120 osd: osd op complaint time: 180 conf: global: bluestore warn on legacy statfs: false bluestore warn on no per pool omap: false mon pg warn min per osd: 0 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 mon down mkfs grace: 300 mon warn on osd down out interval zero: false osd: bdev async discard: true bdev enable discard: true bluestore allocator: bitmap bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - FS_DEGRADED - fs.*is degraded - filesystem is degraded - FS_INLINE_DATA_DEPRECATED - FS_WITH_FAILED_MDS - MDS_ALL_DOWN - filesystem is offline - MDS_DAMAGE - MDS_DEGRADED - MDS_FAILED - MDS_INSUFFICIENT_STANDBY - insufficient standby MDS daemons available - MDS_UP_LESS_THAN_MAX - online, but wants - filesystem is online with fewer MDS than max_mds - POOL_APP_NOT_ENABLED - do not have an application enabled - overall HEALTH_ - Replacing daemon - deprecated feature inline_data - overall HEALTH_ - \(OSD_DOWN\) - \(OSD_ - but it is still running - is not responding - PG_AVAILABILITY - PG_DEGRADED - Reduced data availability - Degraded data redundancy - scrub mismatch - ScrubResult - wrongly marked - \(POOL_APP_NOT_ENABLED\) - \(SLOW_OPS\) - overall HEALTH_ - \(MON_MSGR2_NOT_ENABLED\) - slow request sha1: 1843221895adccfb3b692060251dd6a0183bc410 ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} osd: bdev async discard: true bdev enable discard: true bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs install: ceph: flavor: default sha1: 1843221895adccfb3b692060251dd6a0183bc410 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-pdonnell-testing-20240920.212106-debug sha1: 1843221895adccfb3b692060251dd6a0183bc410 owner: scheduled_pdonnell@teuthology priority: 90 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mon.b - mon.c - mgr.x - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 seed: 7864 sha1: 1843221895adccfb3b692060251dd6a0183bc410 sleep_before_teardown: 0 subset: 67/512 suite: fs suite_branch: wip-pdonnell-testing-20240920.212106-debug suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_1843221895adccfb3b692060251dd6a0183bc410/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 1843221895adccfb3b692060251dd6a0183bc410 targets: smithi105.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOfZSKdnL6ONyv5UGvKyQeUbyh7h+S0cxRlXb6GX6LKEL3HqXfi8wMvJ901BrKQ1NAm6rq8A5tYAvXo2rq8MnH8= tasks: - install: branch: reef exclude_packages: - librados3 - ceph-mgr-dashboard - ceph-mgr-diskprediction-local - ceph-mgr-rook - ceph-mgr-cephadm - cephadm - ceph-volume extra_packages: - librados2 - print: '**** done installing reef' - ceph: conf: global: mon warn on pool no app: false ms bind msgr2: false log-ignorelist: - overall HEALTH_ - \(FS_ - \(MDS_ - \(OSD_ - \(MON_DOWN\) - \(CACHE_POOL_ - \(POOL_ - \(MGR_DOWN\) - \(PG_ - \(SMALLER_PGP_NUM\) - Monitor daemon marked osd - Behind on trimming - Manager daemon - exec: osd.0: - ceph osd set-require-min-compat-client reef - print: '**** done ceph' - print: '*** upgrading, no cephfs present' - exec: mon.a: - ceph fs dump - install.upgrade: mon.a: null - print: '**** done install.upgrade' - ceph.restart: daemons: - mon.* - mgr.* mon-health-to-clog: false wait-for-healthy: false - ceph.healthy: null - ceph.restart: daemons: - osd.* wait-for-healthy: false wait-for-osds-up: true - exec: mon.a: - ceph versions - ceph osd dump -f json-pretty - ceph fs dump - ceph osd require-osd-release squid - for f in `ceph osd pool ls` ; do ceph osd pool set $f pg_autoscale_mode off ; done - ceph.healthy: null - print: '**** done ceph.restart' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-20_23:48:22 tube: smithi user: pdonnell verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2902 2024-09-21T01:35:50.318 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_1843221895adccfb3b692060251dd6a0183bc410/qa; will attempt to use it 2024-09-21T01:35:50.319 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_1843221895adccfb3b692060251dd6a0183bc410/qa/tasks 2024-09-21T01:35:50.319 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-21T01:35:50.321 INFO:teuthology.task.internal:Checking packages... 2024-09-21T01:35:50.341 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '1843221895adccfb3b692060251dd6a0183bc410' 2024-09-21T01:35:50.341 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T01:35:50.342 INFO:teuthology.packaging:ref: None 2024-09-21T01:35:50.342 INFO:teuthology.packaging:tag: None 2024-09-21T01:35:50.342 INFO:teuthology.packaging:branch: wip-pdonnell-testing-20240920.212106-debug 2024-09-21T01:35:50.342 INFO:teuthology.packaging:sha1: 1843221895adccfb3b692060251dd6a0183bc410 2024-09-21T01:35:50.342 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-pdonnell-testing-20240920.212106-debug 2024-09-21T01:35:50.489 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-5120.g18432218 2024-09-21T01:35:50.492 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-21T01:35:50.498 INFO:teuthology.task.internal:no buildpackages task found 2024-09-21T01:35:50.499 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-21T01:35:50.529 INFO:teuthology.task.internal:Saving configuration 2024-09-21T01:35:50.548 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-21T01:35:50.553 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-21T01:35:50.579 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi105.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/pdonnell-2024-09-20_23:48:22-fs-wip-pdonnell-testing-20240920.212106-debug-distro-default-smithi/7913395', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-21 01:30:57.155853', 'locked_by': 'scheduled_pdonnell@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOfZSKdnL6ONyv5UGvKyQeUbyh7h+S0cxRlXb6GX6LKEL3HqXfi8wMvJ901BrKQ1NAm6rq8A5tYAvXo2rq8MnH8='} 2024-09-21T01:35:50.580 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-21T01:35:50.585 INFO:teuthology.task.internal:roles: ubuntu@smithi105.front.sepia.ceph.com - ['mon.a', 'mon.b', 'mon.c', 'mgr.x', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3'] 2024-09-21T01:35:50.585 INFO:teuthology.run_tasks:Running task console_log... 2024-09-21T01:35:50.633 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f8061290160>, signals=[15]) 2024-09-21T01:35:50.633 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-21T01:35:50.638 INFO:teuthology.task.internal:Opening connections... 2024-09-21T01:35:50.638 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi105.front.sepia.ceph.com 2024-09-21T01:35:50.640 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T01:35:50.746 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-21T01:35:50.758 DEBUG:teuthology.orchestra.run.smithi105:> uname -m 2024-09-21T01:35:50.828 INFO:teuthology.orchestra.run.smithi105.stdout:x86_64 2024-09-21T01:35:50.829 DEBUG:teuthology.orchestra.run.smithi105:> cat /etc/os-release 2024-09-21T01:35:50.886 INFO:teuthology.orchestra.run.smithi105.stdout:NAME="CentOS Stream" 2024-09-21T01:35:50.886 INFO:teuthology.orchestra.run.smithi105.stdout:VERSION="9" 2024-09-21T01:35:50.886 INFO:teuthology.orchestra.run.smithi105.stdout:ID="centos" 2024-09-21T01:35:50.886 INFO:teuthology.orchestra.run.smithi105.stdout:ID_LIKE="rhel fedora" 2024-09-21T01:35:50.886 INFO:teuthology.orchestra.run.smithi105.stdout:VERSION_ID="9" 2024-09-21T01:35:50.886 INFO:teuthology.orchestra.run.smithi105.stdout:PLATFORM_ID="platform:el9" 2024-09-21T01:35:50.886 INFO:teuthology.orchestra.run.smithi105.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-21T01:35:50.886 INFO:teuthology.orchestra.run.smithi105.stdout:ANSI_COLOR="0;31" 2024-09-21T01:35:50.886 INFO:teuthology.orchestra.run.smithi105.stdout:LOGO="fedora-logo-icon" 2024-09-21T01:35:50.886 INFO:teuthology.orchestra.run.smithi105.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-21T01:35:50.887 INFO:teuthology.orchestra.run.smithi105.stdout:HOME_URL="https://centos.org/" 2024-09-21T01:35:50.887 INFO:teuthology.orchestra.run.smithi105.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-21T01:35:50.887 INFO:teuthology.orchestra.run.smithi105.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-21T01:35:50.887 INFO:teuthology.orchestra.run.smithi105.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-21T01:35:50.888 INFO:teuthology.lock.ops:Updating smithi105.front.sepia.ceph.com on lock server 2024-09-21T01:35:50.915 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-21T01:35:50.923 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-21T01:35:50.980 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-21T01:35:50.980 DEBUG:teuthology.orchestra.run.smithi105:> test '!' -e /home/ubuntu/cephtest 2024-09-21T01:35:50.996 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-21T01:35:51.041 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-21T01:35:51.042 DEBUG:teuthology.orchestra.run.smithi105:> test -z $(ls -A /var/lib/ceph) 2024-09-21T01:35:51.067 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-21T01:35:51.119 INFO:teuthology.run_tasks:Running task kernel... 2024-09-21T01:35:51.177 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-21T01:35:51.177 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-21T01:35:51.177 DEBUG:teuthology.orchestra.run.smithi105:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-21T01:35:51.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T01:35:51.194 DEBUG:teuthology.orchestra.run.smithi105:> uname -r 2024-09-21T01:35:51.252 INFO:teuthology.orchestra.run.smithi105.stdout:5.14.0-503.el9.x86_64 2024-09-21T01:35:51.252 INFO:teuthology.task.kernel:Running kernel on smithi105: 5.14.0-503.el9.x86_64 2024-09-21T01:35:51.252 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum install -y kernel 2024-09-21T01:35:54.071 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - BaseOS 7.1 MB/s | 8.3 MB 00:01 2024-09-21T01:35:56.230 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - AppStream 19 MB/s | 20 MB 00:01 2024-09-21T01:36:00.615 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - CRB 10 MB/s | 6.5 MB 00:00 2024-09-21T01:36:02.040 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - Extras packages 86 kB/s | 19 kB 00:00 2024-09-21T01:36:03.103 INFO:teuthology.orchestra.run.smithi105.stdout:Extra Packages for Enterprise Linux 24 MB/s | 23 MB 00:00 2024-09-21T01:36:08.409 INFO:teuthology.orchestra.run.smithi105.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-09-21T01:36:10.315 INFO:teuthology.orchestra.run.smithi105.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-21T01:36:10.315 INFO:teuthology.orchestra.run.smithi105.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-21T01:36:10.428 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T01:36:10.435 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:36:10.436 INFO:teuthology.orchestra.run.smithi105.stdout: Package Architecture Version Repository Size 2024-09-21T01:36:10.436 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:36:10.436 INFO:teuthology.orchestra.run.smithi105.stdout:Installing: 2024-09-21T01:36:10.436 INFO:teuthology.orchestra.run.smithi105.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-21T01:36:10.436 INFO:teuthology.orchestra.run.smithi105.stdout:Installing dependencies: 2024-09-21T01:36:10.436 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-21T01:36:10.436 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-21T01:36:10.437 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-21T01:36:10.437 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:36:10.437 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction Summary 2024-09-21T01:36:10.437 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:36:10.437 INFO:teuthology.orchestra.run.smithi105.stdout:Install 4 Packages 2024-09-21T01:36:10.437 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:36:10.437 INFO:teuthology.orchestra.run.smithi105.stdout:Total download size: 79 M 2024-09-21T01:36:10.437 INFO:teuthology.orchestra.run.smithi105.stdout:Installed size: 126 M 2024-09-21T01:36:10.437 INFO:teuthology.orchestra.run.smithi105.stdout:Downloading Packages: 2024-09-21T01:36:11.268 INFO:teuthology.orchestra.run.smithi105.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 63 kB/s | 36 kB 00:00 2024-09-21T01:36:13.477 INFO:teuthology.orchestra.run.smithi105.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 5.6 MB/s | 16 MB 00:02 2024-09-21T01:36:15.160 INFO:teuthology.orchestra.run.smithi105.stdout:(3/4): kernel-modules-core-5.14.0-508.el9.x86_6 7.3 MB/s | 29 MB 00:03 2024-09-21T01:36:17.293 INFO:teuthology.orchestra.run.smithi105.stdout:(4/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 5.2 MB/s | 35 MB 00:06 2024-09-21T01:36:17.294 INFO:teuthology.orchestra.run.smithi105.stdout:-------------------------------------------------------------------------------- 2024-09-21T01:36:17.294 INFO:teuthology.orchestra.run.smithi105.stdout:Total 11 MB/s | 79 MB 00:06 2024-09-21T01:36:17.978 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction check 2024-09-21T01:36:18.334 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction check succeeded. 2024-09-21T01:36:18.375 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction test 2024-09-21T01:36:19.286 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction test succeeded. 2024-09-21T01:36:19.287 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction 2024-09-21T01:36:21.714 INFO:teuthology.orchestra.run.smithi105.stdout: Preparing : 1/1 2024-09-21T01:36:22.369 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-21T01:36:22.452 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T01:36:25.210 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T01:36:25.339 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T01:36:30.177 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T01:36:30.223 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T01:36:35.059 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T01:37:13.531 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T01:37:13.571 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-21T01:37:13.908 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T01:37:13.908 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-21T01:37:13.908 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T01:37:13.908 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T01:37:14.736 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T01:37:14.736 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:37:14.736 INFO:teuthology.orchestra.run.smithi105.stdout:Installed: 2024-09-21T01:37:14.736 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-21T01:37:14.736 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-21T01:37:14.736 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-21T01:37:14.736 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-21T01:37:14.736 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:37:14.737 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T01:37:15.107 DEBUG:teuthology.orchestra.run.smithi105:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T01:37:15.190 INFO:teuthology.orchestra.run.smithi105.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T01:37:15.191 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T01:37:15.191 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-21T01:37:15.191 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-21T01:37:15.191 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-21T01:37:15.191 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-21T01:37:15.191 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi105.front.sepia.ceph.com, path=None, version=distro) 2024-09-21T01:37:15.191 DEBUG:teuthology.orchestra.run.smithi105:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T01:37:15.250 INFO:teuthology.orchestra.run.smithi105.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T01:37:15.250 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T01:37:15.250 DEBUG:teuthology.orchestra.run.smithi105:> sudo rpm -qi grub2-tools 2024-09-21T01:37:15.281 INFO:teuthology.orchestra.run.smithi105.stdout:Name : grub2-tools 2024-09-21T01:37:15.282 INFO:teuthology.orchestra.run.smithi105.stdout:Epoch : 1 2024-09-21T01:37:15.282 INFO:teuthology.orchestra.run.smithi105.stdout:Version : 2.06 2024-09-21T01:37:15.282 INFO:teuthology.orchestra.run.smithi105.stdout:Release : 82.el9 2024-09-21T01:37:15.282 INFO:teuthology.orchestra.run.smithi105.stdout:Architecture: x86_64 2024-09-21T01:37:15.282 INFO:teuthology.orchestra.run.smithi105.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-21T01:37:15.282 INFO:teuthology.orchestra.run.smithi105.stdout:Group : Unspecified 2024-09-21T01:37:15.282 INFO:teuthology.orchestra.run.smithi105.stdout:Size : 8274670 2024-09-21T01:37:15.282 INFO:teuthology.orchestra.run.smithi105.stdout:License : GPLv3+ 2024-09-21T01:37:15.282 INFO:teuthology.orchestra.run.smithi105.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-21T01:37:15.282 INFO:teuthology.orchestra.run.smithi105.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-21T01:37:15.282 INFO:teuthology.orchestra.run.smithi105.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-21T01:37:15.282 INFO:teuthology.orchestra.run.smithi105.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-21T01:37:15.283 INFO:teuthology.orchestra.run.smithi105.stdout:Packager : builder@centos.org 2024-09-21T01:37:15.283 INFO:teuthology.orchestra.run.smithi105.stdout:Vendor : CentOS 2024-09-21T01:37:15.283 INFO:teuthology.orchestra.run.smithi105.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-21T01:37:15.283 INFO:teuthology.orchestra.run.smithi105.stdout:Summary : Support tools for GRUB. 2024-09-21T01:37:15.283 INFO:teuthology.orchestra.run.smithi105.stdout:Description : 2024-09-21T01:37:15.283 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:37:15.283 INFO:teuthology.orchestra.run.smithi105.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-21T01:37:15.283 INFO:teuthology.orchestra.run.smithi105.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-21T01:37:15.283 INFO:teuthology.orchestra.run.smithi105.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-21T01:37:15.283 INFO:teuthology.orchestra.run.smithi105.stdout:hardware devices. 2024-09-21T01:37:15.283 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:37:15.283 INFO:teuthology.orchestra.run.smithi105.stdout:This subpackage provides tools for support of all platforms. 2024-09-21T01:37:15.284 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-21T01:37:15.284 INFO:teuthology.task.kernel:Updating grub on smithi105 to boot 5.14.0-508.el9.x86_64 2024-09-21T01:37:15.284 DEBUG:teuthology.orchestra.run.smithi105:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-21T01:37:15.975 INFO:teuthology.orchestra.run.smithi105.stderr:Generating grub configuration file ... 2024-09-21T01:37:17.299 INFO:teuthology.orchestra.run.smithi105.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-21T01:37:17.332 INFO:teuthology.orchestra.run.smithi105.stderr:done 2024-09-21T01:37:17.334 DEBUG:teuthology.orchestra.run.smithi105:> mktemp 2024-09-21T01:37:17.349 INFO:teuthology.orchestra.run.smithi105.stdout:/tmp/tmp.cbHr9ErzYQ 2024-09-21T01:37:17.349 DEBUG:teuthology.orchestra.run.smithi105:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.cbHr9ErzYQ 2024-09-21T01:37:17.416 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod 0666 /tmp/tmp.cbHr9ErzYQ 2024-09-21T01:37:17.556 DEBUG:teuthology.orchestra.remote:smithi105:/tmp/tmp.cbHr9ErzYQ is 6KB 2024-09-21T01:37:17.604 DEBUG:teuthology.orchestra.run.smithi105:> rm -fr /tmp/tmp.cbHr9ErzYQ 2024-09-21T01:37:17.619 DEBUG:teuthology.orchestra.run.smithi105:> sudo /bin/ls /boot/loader/entries || true 2024-09-21T01:37:17.681 INFO:teuthology.orchestra.run.smithi105.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-21T01:37:17.682 INFO:teuthology.orchestra.run.smithi105.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-21T01:37:17.682 INFO:teuthology.orchestra.run.smithi105.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-21T01:37:17.682 INFO:teuthology.orchestra.run.smithi105.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-21T01:37:17.683 DEBUG:teuthology.orchestra.run.smithi105:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-21T01:37:17.832 DEBUG:teuthology.orchestra.run.smithi105:> sudo shutdown -r now 2024-09-21T01:37:47.848 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-21T01:37:47.848 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-21T01:37:47.849 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T01:38:47.896 DEBUG:teuthology.orchestra.remote:timed out 2024-09-21T01:38:56.902 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-21T01:38:56.903 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T01:38:57.218 DEBUG:teuthology.orchestra.run.smithi105:> true 2024-09-21T01:38:57.635 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-21T01:38:57.635 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-508.el9.x86_64"... 2024-09-21T01:38:57.635 DEBUG:teuthology.orchestra.run.smithi105:> uname -r 2024-09-21T01:38:57.653 INFO:teuthology.orchestra.run.smithi105.stdout:5.14.0-508.el9.x86_64 2024-09-21T01:38:57.653 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-21T01:38:57.653 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-21T01:38:57.653 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-21T01:38:58.654 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-21T01:38:58.654 DEBUG:teuthology.orchestra.run.smithi105:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-21T01:38:58.776 INFO:teuthology.orchestra.run.smithi105.stdout:ttyS1 2024-09-21T01:38:58.801 DEBUG:teuthology.parallel:result is None 2024-09-21T01:38:58.801 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-21T01:38:58.809 INFO:teuthology.task.internal:Creating test directory... 2024-09-21T01:38:58.809 DEBUG:teuthology.orchestra.run.smithi105:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-21T01:38:58.824 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-21T01:38:58.850 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-21T01:38:58.928 INFO:teuthology.task.internal:Creating archive directory... 2024-09-21T01:38:58.928 DEBUG:teuthology.orchestra.run.smithi105:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-21T01:38:58.962 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-21T01:38:59.013 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-21T01:38:59.014 DEBUG:teuthology.orchestra.run.smithi105:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-21T01:38:59.067 INFO:teuthology.orchestra.run.smithi105.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T01:38:59.077 INFO:teuthology.orchestra.run.smithi105.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T01:38:59.078 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-21T01:38:59.091 INFO:teuthology.task.internal:Configuring sudo... 2024-09-21T01:38:59.091 DEBUG:teuthology.orchestra.run.smithi105:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-21T01:38:59.115 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-21T01:38:59.171 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-21T01:38:59.171 DEBUG:teuthology.orchestra.run.smithi105:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-21T01:38:59.186 DEBUG:teuthology.orchestra.run.smithi105:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T01:38:59.270 DEBUG:teuthology.orchestra.run.smithi105:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T01:38:59.312 DEBUG:teuthology.orchestra.run.smithi105:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T01:38:59.394 DEBUG:teuthology.orchestra.run.smithi105:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T01:38:59.435 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:38:59.435 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-21T01:38:59.501 DEBUG:teuthology.orchestra.run.smithi105:> sudo service rsyslog restart 2024-09-21T01:38:59.614 INFO:teuthology.orchestra.run.smithi105.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T01:38:59.967 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-21T01:38:59.980 INFO:teuthology.task.internal:Starting timer... 2024-09-21T01:38:59.980 INFO:teuthology.run_tasks:Running task pcp... 2024-09-21T01:39:00.073 INFO:teuthology.run_tasks:Running task selinux... 2024-09-21T01:39:00.147 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-21T01:39:00.148 DEBUG:teuthology.orchestra.run.smithi105:> sudo service auditd rotate 2024-09-21T01:39:00.219 INFO:teuthology.orchestra.run.smithi105.stdout:Rotating logs: 2024-09-21T01:39:00.221 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-21T01:39:00.221 DEBUG:teuthology.orchestra.run.smithi105:> /usr/sbin/getenforce 2024-09-21T01:39:00.239 INFO:teuthology.orchestra.run.smithi105.stdout:Permissive 2024-09-21T01:39:00.239 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi105.front.sepia.ceph.com': 'permissive'} 2024-09-21T01:39:00.239 DEBUG:teuthology.orchestra.run.smithi105:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T01:39:00.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T01:39:00.304 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-21T01:39:00.304 DEBUG:teuthology.orchestra.run.smithi105:> sudo /usr/sbin/setenforce permissive 2024-09-21T01:39:00.371 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-21T01:39:00.380 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-21T01:39:00.392 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-21T01:39:00.392 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi105.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-21T01:42:45.509 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi105.front.sepia.ceph.com')] 2024-09-21T01:42:45.510 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-21T01:42:45.511 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T01:42:45.586 DEBUG:teuthology.orchestra.run.smithi105:> true 2024-09-21T01:42:45.662 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-21T01:42:45.662 INFO:teuthology.run_tasks:Running task clock... 2024-09-21T01:42:45.674 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-21T01:42:45.674 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-21T01:42:45.675 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T01:42:45.732 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-21T01:42:45.747 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-21T01:42:45.771 INFO:teuthology.orchestra.run.smithi105.stderr:sudo: ntpd: command not found 2024-09-21T01:42:45.782 INFO:teuthology.orchestra.run.smithi105.stdout:506 Cannot talk to daemon 2024-09-21T01:42:45.796 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-21T01:42:45.811 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-21T01:42:45.871 INFO:teuthology.orchestra.run.smithi105.stderr:bash: line 1: ntpq: command not found 2024-09-21T01:42:45.876 INFO:teuthology.orchestra.run.smithi105.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T01:42:45.876 INFO:teuthology.orchestra.run.smithi105.stdout:=============================================================================== 2024-09-21T01:42:45.876 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T01:42:45.876 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T01:42:45.876 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T01:42:45.876 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T01:42:45.876 INFO:teuthology.run_tasks:Running task install... 2024-09-21T01:42:45.885 DEBUG:teuthology.task.install:project ceph 2024-09-21T01:42:45.885 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '1843221895adccfb3b692060251dd6a0183bc410'}} 2024-09-21T01:42:45.885 DEBUG:teuthology.task.install:config {'branch': 'reef', 'exclude_packages': ['librados3', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'cephadm', 'ceph-volume'], 'extra_packages': ['librados2'], 'flavor': 'default', 'sha1': '1843221895adccfb3b692060251dd6a0183bc410'} 2024-09-21T01:42:45.885 INFO:teuthology.task.install:Using flavor: default 2024-09-21T01:42:45.890 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-21T01:42:45.890 INFO:teuthology.task.install:extra packages: ['librados2'] 2024-09-21T01:42:45.891 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'reef', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['librados3', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'cephadm', 'ceph-volume'], 'extra_packages': ['librados2'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '1843221895adccfb3b692060251dd6a0183bc410', 'tag': None, 'wait_for_package': False} 2024-09-21T01:42:45.891 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T01:42:45.891 INFO:teuthology.packaging:ref: None 2024-09-21T01:42:45.891 INFO:teuthology.packaging:tag: None 2024-09-21T01:42:45.891 INFO:teuthology.packaging:branch: reef 2024-09-21T01:42:45.891 INFO:teuthology.packaging:sha1: 1843221895adccfb3b692060251dd6a0183bc410 2024-09-21T01:42:45.891 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-21T01:42:46.138 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/ 2024-09-21T01:42:46.138 INFO:teuthology.task.install.rpm:Package version is 18.2.4-871.g6a1bdf85 2024-09-21T01:42:46.314 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/reef/6a1bdf854bb5803efd58957907442aa01f41a8fc/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-21T01:42:46.314 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:42:46.314 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-21T01:42:46.342 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, ceph-immutable-object-cache, ceph-mgr, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, librados2 on remote rpm x86_64 2024-09-21T01:42:46.342 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T01:42:46.342 INFO:teuthology.packaging:ref: None 2024-09-21T01:42:46.342 INFO:teuthology.packaging:tag: None 2024-09-21T01:42:46.342 INFO:teuthology.packaging:branch: reef 2024-09-21T01:42:46.342 INFO:teuthology.packaging:sha1: 1843221895adccfb3b692060251dd6a0183bc410 2024-09-21T01:42:46.343 DEBUG:teuthology.orchestra.run.smithi105:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/reef/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-21T01:42:46.409 DEBUG:teuthology.orchestra.run.smithi105:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-21T01:42:46.485 DEBUG:teuthology.orchestra.run.smithi105:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-21T01:42:46.549 INFO:teuthology.orchestra.run.smithi105.stdout:check_obsoletes = 1 2024-09-21T01:42:46.551 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum clean all 2024-09-21T01:42:46.776 INFO:teuthology.orchestra.run.smithi105.stdout:45 files removed 2024-09-21T01:42:46.803 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y install ceph-radosgw 2024-09-21T01:42:47.419 INFO:teuthology.orchestra.run.smithi105.stdout:ceph packages for x86_64 231 kB/s | 77 kB 00:00 2024-09-21T01:42:47.770 INFO:teuthology.orchestra.run.smithi105.stdout:ceph noarch packages 37 kB/s | 11 kB 00:00 2024-09-21T01:42:48.220 INFO:teuthology.orchestra.run.smithi105.stdout:ceph source packages 4.6 kB/s | 1.9 kB 00:00 2024-09-21T01:42:48.812 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - BaseOS 15 MB/s | 8.3 MB 00:00 2024-09-21T01:42:51.588 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - AppStream 12 MB/s | 20 MB 00:01 2024-09-21T01:42:57.039 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - CRB 3.9 MB/s | 6.5 MB 00:01 2024-09-21T01:42:59.082 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - Extras packages 28 kB/s | 19 kB 00:00 2024-09-21T01:43:00.203 INFO:teuthology.orchestra.run.smithi105.stdout:Extra Packages for Enterprise Linux 23 MB/s | 23 MB 00:00 2024-09-21T01:43:05.650 INFO:teuthology.orchestra.run.smithi105.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-09-21T01:43:07.884 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T01:43:07.913 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:43:07.913 INFO:teuthology.orchestra.run.smithi105.stdout: Package Arch Version Repository Size 2024-09-21T01:43:07.913 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:43:07.913 INFO:teuthology.orchestra.run.smithi105.stdout:Installing: 2024-09-21T01:43:07.913 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-radosgw x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 7.7 M 2024-09-21T01:43:07.913 INFO:teuthology.orchestra.run.smithi105.stdout:Upgrading: 2024-09-21T01:43:07.913 INFO:teuthology.orchestra.run.smithi105.stdout: librados2 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 3.3 M 2024-09-21T01:43:07.913 INFO:teuthology.orchestra.run.smithi105.stdout: librbd1 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 3.0 M 2024-09-21T01:43:07.913 INFO:teuthology.orchestra.run.smithi105.stdout:Installing dependencies: 2024-09-21T01:43:07.914 INFO:teuthology.orchestra.run.smithi105.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-09-21T01:43:07.914 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-base x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 5.1 M 2024-09-21T01:43:07.914 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-common x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 18 M 2024-09-21T01:43:07.914 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-selinux x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 25 k 2024-09-21T01:43:07.914 INFO:teuthology.orchestra.run.smithi105.stdout: fmt x86_64 8.1.1-5.el9 epel 111 k 2024-09-21T01:43:07.914 INFO:teuthology.orchestra.run.smithi105.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-21T01:43:07.914 INFO:teuthology.orchestra.run.smithi105.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-09-21T01:43:07.914 INFO:teuthology.orchestra.run.smithi105.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-09-21T01:43:07.914 INFO:teuthology.orchestra.run.smithi105.stdout: libcephfs2 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 711 k 2024-09-21T01:43:07.914 INFO:teuthology.orchestra.run.smithi105.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-21T01:43:07.914 INFO:teuthology.orchestra.run.smithi105.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-21T01:43:07.914 INFO:teuthology.orchestra.run.smithi105.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-21T01:43:07.914 INFO:teuthology.orchestra.run.smithi105.stdout: libradosstriper1 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 477 k 2024-09-21T01:43:07.914 INFO:teuthology.orchestra.run.smithi105.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-21T01:43:07.915 INFO:teuthology.orchestra.run.smithi105.stdout: librgw2 x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 4.5 M 2024-09-21T01:43:07.915 INFO:teuthology.orchestra.run.smithi105.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-21T01:43:07.915 INFO:teuthology.orchestra.run.smithi105.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-21T01:43:07.915 INFO:teuthology.orchestra.run.smithi105.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-09-21T01:43:07.915 INFO:teuthology.orchestra.run.smithi105.stdout: python3-ceph-argparse x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 45 k 2024-09-21T01:43:07.915 INFO:teuthology.orchestra.run.smithi105.stdout: python3-ceph-common x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 129 k 2024-09-21T01:43:07.915 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cephfs x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 161 k 2024-09-21T01:43:07.915 INFO:teuthology.orchestra.run.smithi105.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-21T01:43:07.915 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rados x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 320 k 2024-09-21T01:43:07.915 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rbd x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 300 k 2024-09-21T01:43:07.915 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rgw x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 99 k 2024-09-21T01:43:07.915 INFO:teuthology.orchestra.run.smithi105.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2024-09-21T01:43:07.915 INFO:teuthology.orchestra.run.smithi105.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-21T01:43:07.915 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:07.916 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction Summary 2024-09-21T01:43:07.916 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:43:07.916 INFO:teuthology.orchestra.run.smithi105.stdout:Install 28 Packages 2024-09-21T01:43:07.916 INFO:teuthology.orchestra.run.smithi105.stdout:Upgrade 2 Packages 2024-09-21T01:43:07.916 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:07.916 INFO:teuthology.orchestra.run.smithi105.stdout:Total download size: 53 M 2024-09-21T01:43:07.916 INFO:teuthology.orchestra.run.smithi105.stdout:Downloading Packages: 2024-09-21T01:43:08.960 INFO:teuthology.orchestra.run.smithi105.stdout:(1/30): ceph-base-18.2.4-871.g6a1bdf85.el9.x86_ 9.2 MB/s | 5.1 MB 00:00 2024-09-21T01:43:09.069 INFO:teuthology.orchestra.run.smithi105.stdout:(2/30): ceph-radosgw-18.2.4-871.g6a1bdf85.el9.x 12 MB/s | 7.7 MB 00:00 2024-09-21T01:43:09.094 INFO:teuthology.orchestra.run.smithi105.stdout:(3/30): ceph-selinux-18.2.4-871.g6a1bdf85.el9.x 188 kB/s | 25 kB 00:00 2024-09-21T01:43:09.135 INFO:teuthology.orchestra.run.smithi105.stdout:(4/30): libcephfs2-18.2.4-871.g6a1bdf85.el9.x86 10 MB/s | 711 kB 00:00 2024-09-21T01:43:09.161 INFO:teuthology.orchestra.run.smithi105.stdout:(5/30): libradosstriper1-18.2.4-871.g6a1bdf85.e 7.0 MB/s | 477 kB 00:00 2024-09-21T01:43:09.219 INFO:teuthology.orchestra.run.smithi105.stdout:(6/30): python3-ceph-argparse-18.2.4-871.g6a1bd 770 kB/s | 45 kB 00:00 2024-09-21T01:43:09.287 INFO:teuthology.orchestra.run.smithi105.stdout:(7/30): python3-ceph-common-18.2.4-871.g6a1bdf8 1.9 MB/s | 129 kB 00:00 2024-09-21T01:43:09.387 INFO:teuthology.orchestra.run.smithi105.stdout:(8/30): librgw2-18.2.4-871.g6a1bdf85.el9.x86_64 18 MB/s | 4.5 MB 00:00 2024-09-21T01:43:09.412 INFO:teuthology.orchestra.run.smithi105.stdout:(9/30): python3-cephfs-18.2.4-871.g6a1bdf85.el9 1.3 MB/s | 161 kB 00:00 2024-09-21T01:43:09.604 INFO:teuthology.orchestra.run.smithi105.stdout:(10/30): ceph-common-18.2.4-871.g6a1bdf85.el9.x 15 MB/s | 18 MB 00:01 2024-09-21T01:43:09.629 INFO:teuthology.orchestra.run.smithi105.stdout:(11/30): python3-rados-18.2.4-871.g6a1bdf85.el9 1.3 MB/s | 320 kB 00:00 2024-09-21T01:43:09.654 INFO:teuthology.orchestra.run.smithi105.stdout:(12/30): python3-rgw-18.2.4-871.g6a1bdf85.el9.x 1.9 MB/s | 99 kB 00:00 2024-09-21T01:43:09.680 INFO:teuthology.orchestra.run.smithi105.stdout:(13/30): python3-rbd-18.2.4-871.g6a1bdf85.el9.x 1.1 MB/s | 300 kB 00:00 2024-09-21T01:43:09.780 INFO:teuthology.orchestra.run.smithi105.stdout:(14/30): librabbitmq-0.11.0-7.el9.x86_64.rpm 453 kB/s | 45 kB 00:00 2024-09-21T01:43:09.805 INFO:teuthology.orchestra.run.smithi105.stdout:(15/30): boost-program-options-1.75.0-8.el9.x86 608 kB/s | 107 kB 00:00 2024-09-21T01:43:09.831 INFO:teuthology.orchestra.run.smithi105.stdout:(16/30): libpmemobj-1.12.1-1.el9.x86_64.rpm 911 kB/s | 160 kB 00:00 2024-09-21T01:43:09.873 INFO:teuthology.orchestra.run.smithi105.stdout:(17/30): python3-prettytable-0.7.2-27.el9.noarc 1.0 MB/s | 42 kB 00:00 2024-09-21T01:43:09.923 INFO:teuthology.orchestra.run.smithi105.stdout:(18/30): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.4 MB/s | 292 kB 00:00 2024-09-21T01:43:09.965 INFO:teuthology.orchestra.run.smithi105.stdout:(19/30): fmt-8.1.1-5.el9.x86_64.rpm 1.2 MB/s | 111 kB 00:00 2024-09-21T01:43:10.024 INFO:teuthology.orchestra.run.smithi105.stdout:(20/30): gperftools-libs-2.9.1-3.el9.x86_64.rpm 3.0 MB/s | 308 kB 00:00 2024-09-21T01:43:10.057 INFO:teuthology.orchestra.run.smithi105.stdout:(21/30): librdkafka-1.6.1-102.el9.x86_64.rpm 2.3 MB/s | 662 kB 00:00 2024-09-21T01:43:10.074 INFO:teuthology.orchestra.run.smithi105.stdout:(22/30): libarrow-doc-9.0.0-11.el9.noarch.rpm 517 kB/s | 26 kB 00:00 2024-09-21T01:43:10.108 INFO:teuthology.orchestra.run.smithi105.stdout:(23/30): libunwind-1.6.2-1.el9.x86_64.rpm 2.5 MB/s | 67 kB 00:00 2024-09-21T01:43:10.133 INFO:teuthology.orchestra.run.smithi105.stdout:(24/30): liboath-2.6.7-2.el9.x86_64.rpm 643 kB/s | 49 kB 00:00 2024-09-21T01:43:10.175 INFO:teuthology.orchestra.run.smithi105.stdout:(25/30): re2-20211101-20.el9.x86_64.rpm 4.5 MB/s | 191 kB 00:00 2024-09-21T01:43:10.242 INFO:teuthology.orchestra.run.smithi105.stdout:(26/30): libarrow-9.0.0-11.el9.x86_64.rpm 16 MB/s | 4.4 MB 00:00 2024-09-21T01:43:10.276 INFO:teuthology.orchestra.run.smithi105.stdout:(27/30): parquet-libs-9.0.0-11.el9.x86_64.rpm 4.9 MB/s | 839 kB 00:00 2024-09-21T01:43:10.334 INFO:teuthology.orchestra.run.smithi105.stdout:(28/30): thrift-0.15.0-2.el9.x86_64.rpm 10 MB/s | 1.6 MB 00:00 2024-09-21T01:43:10.476 INFO:teuthology.orchestra.run.smithi105.stdout:(29/30): librados2-18.2.4-871.g6a1bdf85.el9.x86 14 MB/s | 3.3 MB 00:00 2024-09-21T01:43:10.534 INFO:teuthology.orchestra.run.smithi105.stdout:(30/30): librbd1-18.2.4-871.g6a1bdf85.el9.x86_6 12 MB/s | 3.0 MB 00:00 2024-09-21T01:43:10.539 INFO:teuthology.orchestra.run.smithi105.stdout:-------------------------------------------------------------------------------- 2024-09-21T01:43:10.539 INFO:teuthology.orchestra.run.smithi105.stdout:Total 20 MB/s | 53 MB 00:02 2024-09-21T01:43:10.607 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction check 2024-09-21T01:43:10.642 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction check succeeded. 2024-09-21T01:43:10.642 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction test 2024-09-21T01:43:11.235 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction test succeeded. 2024-09-21T01:43:11.235 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction 2024-09-21T01:43:12.610 INFO:teuthology.orchestra.run.smithi105.stdout: Preparing : 1/1 2024-09-21T01:43:12.696 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/32 2024-09-21T01:43:12.732 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/32 2024-09-21T01:43:12.781 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/32 2024-09-21T01:43:13.092 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/32 2024-09-21T01:43:13.197 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/32 2024-09-21T01:43:13.297 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/32 2024-09-21T01:43:13.382 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 6/32 2024-09-21T01:43:13.415 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 7/32 2024-09-21T01:43:13.478 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 7/32 2024-09-21T01:43:13.515 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : liboath-2.6.7-2.el9.x86_64 8/32 2024-09-21T01:43:13.559 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-ceph-argparse-2:18.2.4-871.g6a1bdf85.el9.x 9/32 2024-09-21T01:43:13.625 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 10/32 2024-09-21T01:43:13.651 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 11/32 2024-09-21T01:43:13.717 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 11/32 2024-09-21T01:43:13.822 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/32 2024-09-21T01:43:13.859 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 13/32 2024-09-21T01:43:13.916 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/32 2024-09-21T01:43:13.956 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/32 2024-09-21T01:43:13.986 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 16/32 2024-09-21T01:43:14.019 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : fmt-8.1.1-5.el9.x86_64 17/32 2024-09-21T01:43:14.073 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 18/32 2024-09-21T01:43:14.373 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 19/32 2024-09-21T01:43:14.415 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 20/32 2024-09-21T01:43:14.489 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 20/32 2024-09-21T01:43:14.520 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 21/32 2024-09-21T01:43:14.555 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 22/32 2024-09-21T01:43:14.959 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 23/32 2024-09-21T01:43:15.001 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 24/32 2024-09-21T01:43:15.077 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 24/32 2024-09-21T01:43:15.150 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 25/32 2024-09-21T01:43:15.187 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86 26/32 2024-09-21T01:43:17.310 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 27/32 2024-09-21T01:43:17.350 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 27/32 2024-09-21T01:43:17.885 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 27/32 2024-09-21T01:43:17.920 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 28/32 2024-09-21T01:43:18.179 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 28/32 2024-09-21T01:43:18.179 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-21T01:43:18.179 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-21T01:43:18.179 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:18.271 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/32 2024-09-21T01:43:28.029 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/32 2024-09-21T01:43:28.029 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /sys 2024-09-21T01:43:28.029 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /proc 2024-09-21T01:43:28.029 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /mnt 2024-09-21T01:43:28.029 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /var/tmp 2024-09-21T01:43:28.029 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /home 2024-09-21T01:43:28.029 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /root 2024-09-21T01:43:28.029 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /tmp 2024-09-21T01:43:28.029 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:28.771 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/32 2024-09-21T01:43:28.796 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/32 2024-09-21T01:43:28.796 INFO:teuthology.orchestra.run.smithi105.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T01:43:28.796 INFO:teuthology.orchestra.run.smithi105.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-21T01:43:28.796 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T01:43:28.796 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T01:43:28.796 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:28.798 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-21T01:43:28.882 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-21T01:43:28.882 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-21T01:43:30.018 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-21T01:43:30.018 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/32 2024-09-21T01:43:30.018 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/32 2024-09-21T01:43:30.018 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/32 2024-09-21T01:43:30.018 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/32 2024-09-21T01:43:30.018 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/32 2024-09-21T01:43:30.018 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 6/32 2024-09-21T01:43:30.018 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 7/32 2024-09-21T01:43:30.018 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-ceph-argparse-2:18.2.4-871.g6a1bdf85.el9.x 8/32 2024-09-21T01:43:30.018 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86 9/32 2024-09-21T01:43:30.018 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 10/32 2024-09-21T01:43:30.018 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 11/32 2024-09-21T01:43:30.019 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 12/32 2024-09-21T01:43:30.019 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 13/32 2024-09-21T01:43:30.019 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/32 2024-09-21T01:43:30.019 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/32 2024-09-21T01:43:30.019 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/32 2024-09-21T01:43:30.019 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/32 2024-09-21T01:43:30.019 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/32 2024-09-21T01:43:30.019 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/32 2024-09-21T01:43:30.019 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : fmt-8.1.1-5.el9.x86_64 20/32 2024-09-21T01:43:30.019 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 21/32 2024-09-21T01:43:30.019 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 22/32 2024-09-21T01:43:30.019 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 23/32 2024-09-21T01:43:30.019 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 24/32 2024-09-21T01:43:30.019 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 25/32 2024-09-21T01:43:30.020 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 26/32 2024-09-21T01:43:30.020 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : re2-1:20211101-20.el9.x86_64 27/32 2024-09-21T01:43:30.020 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 28/32 2024-09-21T01:43:30.020 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/32 2024-09-21T01:43:30.020 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 30/32 2024-09-21T01:43:30.020 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 31/32 2024-09-21T01:43:30.957 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 32/32 2024-09-21T01:43:30.957 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:30.957 INFO:teuthology.orchestra.run.smithi105.stdout:Upgraded: 2024-09-21T01:43:30.957 INFO:teuthology.orchestra.run.smithi105.stdout: librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:30.957 INFO:teuthology.orchestra.run.smithi105.stdout: librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:30.957 INFO:teuthology.orchestra.run.smithi105.stdout:Installed: 2024-09-21T01:43:30.957 INFO:teuthology.orchestra.run.smithi105.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-09-21T01:43:30.957 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:30.957 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:30.957 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:30.957 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:30.957 INFO:teuthology.orchestra.run.smithi105.stdout: fmt-8.1.1-5.el9.x86_64 2024-09-21T01:43:30.958 INFO:teuthology.orchestra.run.smithi105.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-21T01:43:30.958 INFO:teuthology.orchestra.run.smithi105.stdout: libarrow-9.0.0-11.el9.x86_64 2024-09-21T01:43:30.958 INFO:teuthology.orchestra.run.smithi105.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-09-21T01:43:30.958 INFO:teuthology.orchestra.run.smithi105.stdout: libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:30.958 INFO:teuthology.orchestra.run.smithi105.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-21T01:43:30.958 INFO:teuthology.orchestra.run.smithi105.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-21T01:43:30.958 INFO:teuthology.orchestra.run.smithi105.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-21T01:43:30.958 INFO:teuthology.orchestra.run.smithi105.stdout: libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:30.958 INFO:teuthology.orchestra.run.smithi105.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-21T01:43:30.958 INFO:teuthology.orchestra.run.smithi105.stdout: librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:30.958 INFO:teuthology.orchestra.run.smithi105.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-21T01:43:30.958 INFO:teuthology.orchestra.run.smithi105.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-21T01:43:30.959 INFO:teuthology.orchestra.run.smithi105.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-09-21T01:43:30.959 INFO:teuthology.orchestra.run.smithi105.stdout: python3-ceph-argparse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:30.959 INFO:teuthology.orchestra.run.smithi105.stdout: python3-ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:30.959 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:30.959 INFO:teuthology.orchestra.run.smithi105.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-21T01:43:30.959 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:30.959 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:30.959 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:30.959 INFO:teuthology.orchestra.run.smithi105.stdout: re2-1:20211101-20.el9.x86_64 2024-09-21T01:43:30.959 INFO:teuthology.orchestra.run.smithi105.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-21T01:43:30.959 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:30.959 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T01:43:31.374 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y install ceph-test 2024-09-21T01:43:31.963 INFO:teuthology.orchestra.run.smithi105.stdout:Last metadata expiration check: 0:00:26 ago on Sat 21 Sep 2024 01:43:05 AM UTC. 2024-09-21T01:43:32.099 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T01:43:32.100 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:43:32.100 INFO:teuthology.orchestra.run.smithi105.stdout: Package Arch Version Repository Size 2024-09-21T01:43:32.100 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:43:32.100 INFO:teuthology.orchestra.run.smithi105.stdout:Installing: 2024-09-21T01:43:32.100 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-test x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 40 M 2024-09-21T01:43:32.100 INFO:teuthology.orchestra.run.smithi105.stdout:Installing dependencies: 2024-09-21T01:43:32.100 INFO:teuthology.orchestra.run.smithi105.stdout: libcephsqlite x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 166 k 2024-09-21T01:43:32.100 INFO:teuthology.orchestra.run.smithi105.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-21T01:43:32.101 INFO:teuthology.orchestra.run.smithi105.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-21T01:43:32.101 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:32.101 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction Summary 2024-09-21T01:43:32.101 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:43:32.101 INFO:teuthology.orchestra.run.smithi105.stdout:Install 4 Packages 2024-09-21T01:43:32.101 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:32.101 INFO:teuthology.orchestra.run.smithi105.stdout:Total download size: 40 M 2024-09-21T01:43:32.101 INFO:teuthology.orchestra.run.smithi105.stdout:Installed size: 164 M 2024-09-21T01:43:32.101 INFO:teuthology.orchestra.run.smithi105.stdout:Downloading Packages: 2024-09-21T01:43:32.503 INFO:teuthology.orchestra.run.smithi105.stdout:(1/4): libcephsqlite-18.2.4-871.g6a1bdf85.el9.x 789 kB/s | 166 kB 00:00 2024-09-21T01:43:33.112 INFO:teuthology.orchestra.run.smithi105.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 105 kB/s | 64 kB 00:00 2024-09-21T01:43:33.229 INFO:teuthology.orchestra.run.smithi105.stdout:(3/4): socat-1.7.4.1-6.el9.x86_64.rpm 325 kB/s | 304 kB 00:00 2024-09-21T01:43:33.587 INFO:teuthology.orchestra.run.smithi105.stdout:(4/4): ceph-test-18.2.4-871.g6a1bdf85.el9.x86_6 31 MB/s | 40 MB 00:01 2024-09-21T01:43:33.590 INFO:teuthology.orchestra.run.smithi105.stdout:-------------------------------------------------------------------------------- 2024-09-21T01:43:33.590 INFO:teuthology.orchestra.run.smithi105.stdout:Total 27 MB/s | 40 MB 00:01 2024-09-21T01:43:33.617 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction check 2024-09-21T01:43:33.637 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction check succeeded. 2024-09-21T01:43:33.637 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction test 2024-09-21T01:43:33.956 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction test succeeded. 2024-09-21T01:43:33.956 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction 2024-09-21T01:43:34.429 INFO:teuthology.orchestra.run.smithi105.stdout: Preparing : 1/1 2024-09-21T01:43:34.506 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-21T01:43:34.545 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-21T01:43:34.572 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/4 2024-09-21T01:43:38.251 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/4 2024-09-21T01:43:38.329 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/4 2024-09-21T01:43:40.337 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/4 2024-09-21T01:43:40.337 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/4 2024-09-21T01:43:40.337 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/4 2024-09-21T01:43:40.337 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-21T01:43:40.662 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-21T01:43:40.662 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:40.662 INFO:teuthology.orchestra.run.smithi105.stdout:Installed: 2024-09-21T01:43:40.662 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:40.662 INFO:teuthology.orchestra.run.smithi105.stdout: libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:40.662 INFO:teuthology.orchestra.run.smithi105.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-21T01:43:40.662 INFO:teuthology.orchestra.run.smithi105.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-21T01:43:40.662 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:40.662 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T01:43:40.990 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y install ceph 2024-09-21T01:43:41.578 INFO:teuthology.orchestra.run.smithi105.stdout:Last metadata expiration check: 0:00:36 ago on Sat 21 Sep 2024 01:43:05 AM UTC. 2024-09-21T01:43:41.718 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T01:43:41.720 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:43:41.720 INFO:teuthology.orchestra.run.smithi105.stdout: Package Arch Version Repository Size 2024-09-21T01:43:41.720 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:43:41.720 INFO:teuthology.orchestra.run.smithi105.stdout:Installing: 2024-09-21T01:43:41.720 INFO:teuthology.orchestra.run.smithi105.stdout: ceph x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 6.5 k 2024-09-21T01:43:41.721 INFO:teuthology.orchestra.run.smithi105.stdout:Installing dependencies: 2024-09-21T01:43:41.721 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mds x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 2.1 M 2024-09-21T01:43:41.721 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 1.5 M 2024-09-21T01:43:41.721 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-modules-core noarch 2:18.2.4-871.g6a1bdf85.el9 ceph-noarch 246 k 2024-09-21T01:43:41.721 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mon x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 4.7 M 2024-09-21T01:43:41.721 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-osd x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 17 M 2024-09-21T01:43:41.721 INFO:teuthology.orchestra.run.smithi105.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-21T01:43:41.721 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-21T01:43:41.721 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-21T01:43:41.721 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-21T01:43:41.721 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-21T01:43:41.721 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-21T01:43:41.721 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-21T01:43:41.721 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-21T01:43:41.722 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-21T01:43:41.722 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-21T01:43:41.722 INFO:teuthology.orchestra.run.smithi105.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-21T01:43:41.722 INFO:teuthology.orchestra.run.smithi105.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-21T01:43:41.722 INFO:teuthology.orchestra.run.smithi105.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-21T01:43:41.722 INFO:teuthology.orchestra.run.smithi105.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-21T01:43:41.722 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-21T01:43:41.722 INFO:teuthology.orchestra.run.smithi105.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-21T01:43:41.722 INFO:teuthology.orchestra.run.smithi105.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-21T01:43:41.722 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-21T01:43:41.722 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-21T01:43:41.722 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-21T01:43:41.722 INFO:teuthology.orchestra.run.smithi105.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-21T01:43:41.723 INFO:teuthology.orchestra.run.smithi105.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-21T01:43:41.723 INFO:teuthology.orchestra.run.smithi105.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-21T01:43:41.723 INFO:teuthology.orchestra.run.smithi105.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-21T01:43:41.723 INFO:teuthology.orchestra.run.smithi105.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-21T01:43:41.723 INFO:teuthology.orchestra.run.smithi105.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-21T01:43:41.723 INFO:teuthology.orchestra.run.smithi105.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-21T01:43:41.723 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:41.723 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction Summary 2024-09-21T01:43:41.723 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:43:41.723 INFO:teuthology.orchestra.run.smithi105.stdout:Install 33 Packages 2024-09-21T01:43:41.723 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:41.724 INFO:teuthology.orchestra.run.smithi105.stdout:Total download size: 29 M 2024-09-21T01:43:41.724 INFO:teuthology.orchestra.run.smithi105.stdout:Installed size: 104 M 2024-09-21T01:43:41.724 INFO:teuthology.orchestra.run.smithi105.stdout:Downloading Packages: 2024-09-21T01:43:42.538 INFO:teuthology.orchestra.run.smithi105.stdout:(1/33): ceph-18.2.4-871.g6a1bdf85.el9.x86_64.rp 27 kB/s | 6.5 kB 00:00 2024-09-21T01:43:42.722 INFO:teuthology.orchestra.run.smithi105.stdout:(2/33): ceph-mgr-18.2.4-871.g6a1bdf85.el9.x86_6 3.4 MB/s | 1.5 MB 00:00 2024-09-21T01:43:42.772 INFO:teuthology.orchestra.run.smithi105.stdout:(3/33): ceph-mds-18.2.4-871.g6a1bdf85.el9.x86_6 4.4 MB/s | 2.1 MB 00:00 2024-09-21T01:43:42.897 INFO:teuthology.orchestra.run.smithi105.stdout:(4/33): ceph-mgr-modules-core-18.2.4-871.g6a1bd 1.9 MB/s | 246 kB 00:00 2024-09-21T01:43:43.073 INFO:teuthology.orchestra.run.smithi105.stdout:(5/33): ceph-mon-18.2.4-871.g6a1bdf85.el9.x86_6 8.8 MB/s | 4.7 MB 00:00 2024-09-21T01:43:43.107 INFO:teuthology.orchestra.run.smithi105.stdout:(6/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.2 MB/s | 253 kB 00:00 2024-09-21T01:43:43.148 INFO:teuthology.orchestra.run.smithi105.stdout:(7/33): python3-ply-3.11-14.el9.noarch.rpm 2.5 MB/s | 106 kB 00:00 2024-09-21T01:43:43.191 INFO:teuthology.orchestra.run.smithi105.stdout:(8/33): python3-pycparser-2.20-6.el9.noarch.rpm 3.2 MB/s | 135 kB 00:00 2024-09-21T01:43:43.232 INFO:teuthology.orchestra.run.smithi105.stdout:(9/33): python3-requests-2.25.1-8.el9.noarch.rp 2.9 MB/s | 125 kB 00:00 2024-09-21T01:43:43.282 INFO:teuthology.orchestra.run.smithi105.stdout:(10/33): python3-urllib3-1.26.5-6.el9.noarch.rp 4.2 MB/s | 215 kB 00:00 2024-09-21T01:43:43.350 INFO:teuthology.orchestra.run.smithi105.stdout:(11/33): python3-cryptography-36.0.1-4.el9.x86_ 4.4 MB/s | 1.2 MB 00:00 2024-09-21T01:43:43.450 INFO:teuthology.orchestra.run.smithi105.stdout:(12/33): python3-mako-1.1.4-6.el9.noarch.rpm 1.0 MB/s | 172 kB 00:00 2024-09-21T01:43:43.467 INFO:teuthology.orchestra.run.smithi105.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 298 kB/s | 35 kB 00:00 2024-09-21T01:43:43.492 INFO:teuthology.orchestra.run.smithi105.stdout:(14/33): python3-pytz-2021.1-5.el9.noarch.rpm 1.2 MB/s | 51 kB 00:00 2024-09-21T01:43:43.526 INFO:teuthology.orchestra.run.smithi105.stdout:(15/33): python3-toml-0.10.2-6.el9.noarch.rpm 709 kB/s | 42 kB 00:00 2024-09-21T01:43:43.559 INFO:teuthology.orchestra.run.smithi105.stdout:(16/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 649 kB/s | 43 kB 00:00 2024-09-21T01:43:43.718 INFO:teuthology.orchestra.run.smithi105.stdout:(17/33): ceph-osd-18.2.4-871.g6a1bdf85.el9.x86_ 17 MB/s | 17 MB 00:00 2024-09-21T01:43:43.760 INFO:teuthology.orchestra.run.smithi105.stdout:(18/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 740 kB/s | 172 kB 00:00 2024-09-21T01:43:43.785 INFO:teuthology.orchestra.run.smithi105.stdout:(19/33): python3-cherrypy-18.6.1-2.el9.noarch.r 1.6 MB/s | 358 kB 00:00 2024-09-21T01:43:43.810 INFO:teuthology.orchestra.run.smithi105.stdout:(20/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 116 kB/s | 11 kB 00:00 2024-09-21T01:43:43.836 INFO:teuthology.orchestra.run.smithi105.stdout:(21/33): python3-jaraco-classes-3.2.1-5.el9.noa 239 kB/s | 18 kB 00:00 2024-09-21T01:43:43.861 INFO:teuthology.orchestra.run.smithi105.stdout:(22/33): python3-jaraco-collections-3.0.0-8.el9 307 kB/s | 23 kB 00:00 2024-09-21T01:43:43.886 INFO:teuthology.orchestra.run.smithi105.stdout:(23/33): python3-jaraco-functools-3.5.0-2.el9.n 257 kB/s | 19 kB 00:00 2024-09-21T01:43:43.911 INFO:teuthology.orchestra.run.smithi105.stdout:(24/33): python3-jaraco-text-3.2.0-6.el9.noarch 261 kB/s | 20 kB 00:00 2024-09-21T01:43:43.937 INFO:teuthology.orchestra.run.smithi105.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 924 kB/s | 46 kB 00:00 2024-09-21T01:43:43.962 INFO:teuthology.orchestra.run.smithi105.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 1.0 MB/s | 79 kB 00:00 2024-09-21T01:43:43.987 INFO:teuthology.orchestra.run.smithi105.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 328 kB/s | 16 kB 00:00 2024-09-21T01:43:44.013 INFO:teuthology.orchestra.run.smithi105.stdout:(28/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.8 MB/s | 90 kB 00:00 2024-09-21T01:43:44.038 INFO:teuthology.orchestra.run.smithi105.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 713 kB/s | 36 kB 00:00 2024-09-21T01:43:44.063 INFO:teuthology.orchestra.run.smithi105.stdout:(30/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.8 MB/s | 272 kB 00:00 2024-09-21T01:43:44.088 INFO:teuthology.orchestra.run.smithi105.stdout:(31/33): python3-webob-1.8.8-2.el9.noarch.rpm 3.0 MB/s | 230 kB 00:00 2024-09-21T01:43:44.114 INFO:teuthology.orchestra.run.smithi105.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 399 kB/s | 20 kB 00:00 2024-09-21T01:43:44.147 INFO:teuthology.orchestra.run.smithi105.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 3.8 MB/s | 427 kB 00:00 2024-09-21T01:43:44.152 INFO:teuthology.orchestra.run.smithi105.stdout:-------------------------------------------------------------------------------- 2024-09-21T01:43:44.152 INFO:teuthology.orchestra.run.smithi105.stdout:Total 12 MB/s | 29 MB 00:02 2024-09-21T01:43:44.275 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction check 2024-09-21T01:43:44.314 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction check succeeded. 2024-09-21T01:43:44.314 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction test 2024-09-21T01:43:44.728 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction test succeeded. 2024-09-21T01:43:44.728 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction 2024-09-21T01:43:46.067 INFO:teuthology.orchestra.run.smithi105.stdout: Preparing : 1/1 2024-09-21T01:43:46.144 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-09-21T01:43:46.176 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-09-21T01:43:46.213 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-09-21T01:43:46.306 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-09-21T01:43:46.391 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-09-21T01:43:46.432 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 6/33 2024-09-21T01:43:46.467 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-09-21T01:43:46.497 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-09-21T01:43:46.532 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-09-21T01:43:46.565 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-09-21T01:43:46.606 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-09-21T01:43:46.649 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-09-21T01:43:46.709 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-09-21T01:43:46.755 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-09-21T01:43:46.828 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-09-21T01:43:46.966 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-09-21T01:43:47.054 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-09-21T01:43:47.107 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 18/33 2024-09-21T01:43:47.147 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-09-21T01:43:47.195 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-09-21T01:43:47.259 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-09-21T01:43:47.453 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-09-21T01:43:47.501 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-09-21T01:43:47.587 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-09-21T01:43:47.685 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-09-21T01:43:47.735 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-09-21T01:43:47.850 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-09-21T01:43:48.012 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-mgr-modules-core-2:18.2.4-871.g6a1bdf85.el9.n 28/33 2024-09-21T01:43:48.037 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/33 2024-09-21T01:43:48.064 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 29/33 2024-09-21T01:43:48.064 INFO:teuthology.orchestra.run.smithi105.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T01:43:48.064 INFO:teuthology.orchestra.run.smithi105.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-21T01:43:48.064 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T01:43:48.064 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T01:43:48.064 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:49.463 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/33 2024-09-21T01:43:49.490 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/33 2024-09-21T01:43:49.490 INFO:teuthology.orchestra.run.smithi105.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T01:43:49.490 INFO:teuthology.orchestra.run.smithi105.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-21T01:43:49.491 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T01:43:49.491 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T01:43:49.491 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:49.939 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 31/33 2024-09-21T01:43:49.963 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 31/33 2024-09-21T01:43:49.963 INFO:teuthology.orchestra.run.smithi105.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T01:43:49.963 INFO:teuthology.orchestra.run.smithi105.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-21T01:43:49.963 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T01:43:49.963 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T01:43:49.963 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:50.173 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 32/33 2024-09-21T01:43:50.198 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 32/33 2024-09-21T01:43:50.198 INFO:teuthology.orchestra.run.smithi105.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T01:43:50.198 INFO:teuthology.orchestra.run.smithi105.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-21T01:43:50.198 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T01:43:50.198 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T01:43:50.198 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:50.411 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 33/33 2024-09-21T01:43:51.961 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 33/33 2024-09-21T01:43:51.961 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/33 2024-09-21T01:43:51.961 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/33 2024-09-21T01:43:51.961 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 3/33 2024-09-21T01:43:51.961 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 4/33 2024-09-21T01:43:51.961 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 5/33 2024-09-21T01:43:51.961 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mgr-modules-core-2:18.2.4-871.g6a1bdf85.el9.n 6/33 2024-09-21T01:43:51.962 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-09-21T01:43:51.962 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-09-21T01:43:51.962 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-09-21T01:43:51.962 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-09-21T01:43:51.962 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-09-21T01:43:51.962 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/33 2024-09-21T01:43:51.962 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-09-21T01:43:51.962 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-09-21T01:43:51.962 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-09-21T01:43:51.962 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-09-21T01:43:51.962 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-09-21T01:43:51.962 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-09-21T01:43:51.962 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-09-21T01:43:51.963 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-09-21T01:43:51.963 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-09-21T01:43:51.963 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-09-21T01:43:51.963 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-09-21T01:43:51.963 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-09-21T01:43:51.963 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-09-21T01:43:51.963 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-09-21T01:43:51.963 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-09-21T01:43:51.963 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-09-21T01:43:51.963 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-09-21T01:43:51.963 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-09-21T01:43:51.963 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 31/33 2024-09-21T01:43:51.963 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-09-21T01:43:53.009 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-09-21T01:43:53.009 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:53.009 INFO:teuthology.orchestra.run.smithi105.stdout:Installed: 2024-09-21T01:43:53.009 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:53.009 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:53.010 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:53.010 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-modules-core-2:18.2.4-871.g6a1bdf85.el9.noarch 2024-09-21T01:43:53.010 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:53.010 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:53.010 INFO:teuthology.orchestra.run.smithi105.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-21T01:43:53.010 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-21T01:43:53.010 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-21T01:43:53.010 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-21T01:43:53.010 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-21T01:43:53.010 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-21T01:43:53.010 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-21T01:43:53.010 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-21T01:43:53.010 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-21T01:43:53.010 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-21T01:43:53.010 INFO:teuthology.orchestra.run.smithi105.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-21T01:43:53.011 INFO:teuthology.orchestra.run.smithi105.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-21T01:43:53.011 INFO:teuthology.orchestra.run.smithi105.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-21T01:43:53.011 INFO:teuthology.orchestra.run.smithi105.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-21T01:43:53.011 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-21T01:43:53.011 INFO:teuthology.orchestra.run.smithi105.stdout: python3-ply-3.11-14.el9.noarch 2024-09-21T01:43:53.011 INFO:teuthology.orchestra.run.smithi105.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-21T01:43:53.011 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-21T01:43:53.011 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-21T01:43:53.011 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-21T01:43:53.011 INFO:teuthology.orchestra.run.smithi105.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-21T01:43:53.011 INFO:teuthology.orchestra.run.smithi105.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-21T01:43:53.011 INFO:teuthology.orchestra.run.smithi105.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-21T01:43:53.011 INFO:teuthology.orchestra.run.smithi105.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-21T01:43:53.011 INFO:teuthology.orchestra.run.smithi105.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-21T01:43:53.012 INFO:teuthology.orchestra.run.smithi105.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-21T01:43:53.012 INFO:teuthology.orchestra.run.smithi105.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-21T01:43:53.012 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:53.012 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T01:43:53.380 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y install ceph-base 2024-09-21T01:43:53.963 INFO:teuthology.orchestra.run.smithi105.stdout:Last metadata expiration check: 0:00:48 ago on Sat 21 Sep 2024 01:43:05 AM UTC. 2024-09-21T01:43:54.050 INFO:teuthology.orchestra.run.smithi105.stdout:Package ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T01:43:54.091 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T01:43:54.092 INFO:teuthology.orchestra.run.smithi105.stdout:Nothing to do. 2024-09-21T01:43:54.092 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T01:43:54.166 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y install ceph-immutable-object-cache 2024-09-21T01:43:54.745 INFO:teuthology.orchestra.run.smithi105.stdout:Last metadata expiration check: 0:00:49 ago on Sat 21 Sep 2024 01:43:05 AM UTC. 2024-09-21T01:43:54.876 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T01:43:54.877 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:43:54.877 INFO:teuthology.orchestra.run.smithi105.stdout: Package Arch Version Repo Size 2024-09-21T01:43:54.877 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:43:54.877 INFO:teuthology.orchestra.run.smithi105.stdout:Installing: 2024-09-21T01:43:54.877 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-immutable-object-cache x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 142 k 2024-09-21T01:43:54.877 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:54.877 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction Summary 2024-09-21T01:43:54.877 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:43:54.877 INFO:teuthology.orchestra.run.smithi105.stdout:Install 1 Package 2024-09-21T01:43:54.877 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:54.877 INFO:teuthology.orchestra.run.smithi105.stdout:Total download size: 142 k 2024-09-21T01:43:54.877 INFO:teuthology.orchestra.run.smithi105.stdout:Installed size: 395 k 2024-09-21T01:43:54.877 INFO:teuthology.orchestra.run.smithi105.stdout:Downloading Packages: 2024-09-21T01:43:55.066 INFO:teuthology.orchestra.run.smithi105.stdout:ceph-immutable-object-cache-18.2.4-871.g6a1bdf8 759 kB/s | 142 kB 00:00 2024-09-21T01:43:55.066 INFO:teuthology.orchestra.run.smithi105.stdout:-------------------------------------------------------------------------------- 2024-09-21T01:43:55.066 INFO:teuthology.orchestra.run.smithi105.stdout:Total 753 kB/s | 142 kB 00:00 2024-09-21T01:43:55.067 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction check 2024-09-21T01:43:55.074 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction check succeeded. 2024-09-21T01:43:55.074 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction test 2024-09-21T01:43:55.130 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction test succeeded. 2024-09-21T01:43:55.130 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction 2024-09-21T01:43:55.273 INFO:teuthology.orchestra.run.smithi105.stdout: Preparing : 1/1 2024-09-21T01:43:55.350 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.e 1/1 2024-09-21T01:43:55.375 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.e 1/1 2024-09-21T01:43:55.375 INFO:teuthology.orchestra.run.smithi105.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T01:43:55.375 INFO:teuthology.orchestra.run.smithi105.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-21T01:43:55.376 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:56.402 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.e 1/1 2024-09-21T01:43:56.403 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:56.403 INFO:teuthology.orchestra.run.smithi105.stdout:Installed: 2024-09-21T01:43:56.403 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:43:56.403 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:56.403 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T01:43:56.644 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y install ceph-mgr 2024-09-21T01:43:57.237 INFO:teuthology.orchestra.run.smithi105.stdout:Last metadata expiration check: 0:00:52 ago on Sat 21 Sep 2024 01:43:05 AM UTC. 2024-09-21T01:43:57.329 INFO:teuthology.orchestra.run.smithi105.stdout:Package ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T01:43:57.370 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T01:43:57.371 INFO:teuthology.orchestra.run.smithi105.stdout:Nothing to do. 2024-09-21T01:43:57.372 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T01:43:57.452 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y install ceph-fuse 2024-09-21T01:43:58.034 INFO:teuthology.orchestra.run.smithi105.stdout:Last metadata expiration check: 0:00:53 ago on Sat 21 Sep 2024 01:43:05 AM UTC. 2024-09-21T01:43:58.165 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T01:43:58.165 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:43:58.165 INFO:teuthology.orchestra.run.smithi105.stdout: Package Arch Version Repository Size 2024-09-21T01:43:58.165 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:43:58.166 INFO:teuthology.orchestra.run.smithi105.stdout:Installing: 2024-09-21T01:43:58.166 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-fuse x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 853 k 2024-09-21T01:43:58.166 INFO:teuthology.orchestra.run.smithi105.stdout:Installing dependencies: 2024-09-21T01:43:58.166 INFO:teuthology.orchestra.run.smithi105.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-21T01:43:58.166 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:58.166 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction Summary 2024-09-21T01:43:58.166 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:43:58.166 INFO:teuthology.orchestra.run.smithi105.stdout:Install 2 Packages 2024-09-21T01:43:58.166 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:43:58.167 INFO:teuthology.orchestra.run.smithi105.stdout:Total download size: 932 k 2024-09-21T01:43:58.167 INFO:teuthology.orchestra.run.smithi105.stdout:Installed size: 2.7 M 2024-09-21T01:43:58.167 INFO:teuthology.orchestra.run.smithi105.stdout:Downloading Packages: 2024-09-21T01:43:58.386 INFO:teuthology.orchestra.run.smithi105.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 771 kB/s | 80 kB 00:00 2024-09-21T01:43:58.586 INFO:teuthology.orchestra.run.smithi105.stdout:(2/2): ceph-fuse-18.2.4-871.g6a1bdf85.el9.x86_6 2.7 MB/s | 853 kB 00:00 2024-09-21T01:43:58.587 INFO:teuthology.orchestra.run.smithi105.stdout:-------------------------------------------------------------------------------- 2024-09-21T01:43:58.587 INFO:teuthology.orchestra.run.smithi105.stdout:Total 2.2 MB/s | 932 kB 00:00 2024-09-21T01:43:58.599 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction check 2024-09-21T01:43:58.620 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction check succeeded. 2024-09-21T01:43:58.620 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction test 2024-09-21T01:43:58.708 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction test succeeded. 2024-09-21T01:43:58.708 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction 2024-09-21T01:43:58.892 INFO:teuthology.orchestra.run.smithi105.stdout: Preparing : 1/1 2024-09-21T01:43:59.019 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-21T01:43:59.094 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/2 2024-09-21T01:43:59.728 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2/2 2024-09-21T01:43:59.787 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/2 2024-09-21T01:44:00.003 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-21T01:44:00.003 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:00.003 INFO:teuthology.orchestra.run.smithi105.stdout:Installed: 2024-09-21T01:44:00.004 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-21T01:44:00.004 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:00.004 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T01:44:00.246 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y install librados-devel 2024-09-21T01:44:00.832 INFO:teuthology.orchestra.run.smithi105.stdout:Last metadata expiration check: 0:00:55 ago on Sat 21 Sep 2024 01:43:05 AM UTC. 2024-09-21T01:44:00.967 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T01:44:00.968 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:44:00.968 INFO:teuthology.orchestra.run.smithi105.stdout: Package Arch Version Repo Size 2024-09-21T01:44:00.968 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:44:00.968 INFO:teuthology.orchestra.run.smithi105.stdout:Installing: 2024-09-21T01:44:00.968 INFO:teuthology.orchestra.run.smithi105.stdout: librados-devel x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 126 k 2024-09-21T01:44:00.968 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:00.969 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction Summary 2024-09-21T01:44:00.969 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:44:00.969 INFO:teuthology.orchestra.run.smithi105.stdout:Install 1 Package 2024-09-21T01:44:00.969 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:00.969 INFO:teuthology.orchestra.run.smithi105.stdout:Total download size: 126 k 2024-09-21T01:44:00.969 INFO:teuthology.orchestra.run.smithi105.stdout:Installed size: 456 k 2024-09-21T01:44:00.969 INFO:teuthology.orchestra.run.smithi105.stdout:Downloading Packages: 2024-09-21T01:44:01.157 INFO:teuthology.orchestra.run.smithi105.stdout:librados-devel-18.2.4-871.g6a1bdf85.el9.x86_64. 673 kB/s | 126 kB 00:00 2024-09-21T01:44:01.158 INFO:teuthology.orchestra.run.smithi105.stdout:-------------------------------------------------------------------------------- 2024-09-21T01:44:01.158 INFO:teuthology.orchestra.run.smithi105.stdout:Total 668 kB/s | 126 kB 00:00 2024-09-21T01:44:01.158 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction check 2024-09-21T01:44:01.163 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction check succeeded. 2024-09-21T01:44:01.163 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction test 2024-09-21T01:44:01.221 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction test succeeded. 2024-09-21T01:44:01.221 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction 2024-09-21T01:44:01.364 INFO:teuthology.orchestra.run.smithi105.stdout: Preparing : 1/1 2024-09-21T01:44:01.460 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T01:44:01.866 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T01:44:02.092 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T01:44:02.092 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:02.092 INFO:teuthology.orchestra.run.smithi105.stdout:Installed: 2024-09-21T01:44:02.092 INFO:teuthology.orchestra.run.smithi105.stdout: librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:44:02.092 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:02.092 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T01:44:02.346 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y install libcephfs2 2024-09-21T01:44:02.940 INFO:teuthology.orchestra.run.smithi105.stdout:Last metadata expiration check: 0:00:57 ago on Sat 21 Sep 2024 01:43:05 AM UTC. 2024-09-21T01:44:03.031 INFO:teuthology.orchestra.run.smithi105.stdout:Package libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T01:44:03.075 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T01:44:03.076 INFO:teuthology.orchestra.run.smithi105.stdout:Nothing to do. 2024-09-21T01:44:03.076 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T01:44:03.151 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y install libcephfs-devel 2024-09-21T01:44:03.728 INFO:teuthology.orchestra.run.smithi105.stdout:Last metadata expiration check: 0:00:58 ago on Sat 21 Sep 2024 01:43:05 AM UTC. 2024-09-21T01:44:03.859 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T01:44:03.860 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:44:03.860 INFO:teuthology.orchestra.run.smithi105.stdout: Package Arch Version Repo Size 2024-09-21T01:44:03.860 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:44:03.861 INFO:teuthology.orchestra.run.smithi105.stdout:Installing: 2024-09-21T01:44:03.861 INFO:teuthology.orchestra.run.smithi105.stdout: libcephfs-devel x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 31 k 2024-09-21T01:44:03.861 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:03.861 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction Summary 2024-09-21T01:44:03.861 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:44:03.861 INFO:teuthology.orchestra.run.smithi105.stdout:Install 1 Package 2024-09-21T01:44:03.861 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:03.861 INFO:teuthology.orchestra.run.smithi105.stdout:Total download size: 31 k 2024-09-21T01:44:03.861 INFO:teuthology.orchestra.run.smithi105.stdout:Installed size: 139 k 2024-09-21T01:44:03.861 INFO:teuthology.orchestra.run.smithi105.stdout:Downloading Packages: 2024-09-21T01:44:04.011 INFO:teuthology.orchestra.run.smithi105.stdout:libcephfs-devel-18.2.4-871.g6a1bdf85.el9.x86_64 207 kB/s | 31 kB 00:00 2024-09-21T01:44:04.012 INFO:teuthology.orchestra.run.smithi105.stdout:-------------------------------------------------------------------------------- 2024-09-21T01:44:04.012 INFO:teuthology.orchestra.run.smithi105.stdout:Total 205 kB/s | 31 kB 00:00 2024-09-21T01:44:04.012 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction check 2024-09-21T01:44:04.016 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction check succeeded. 2024-09-21T01:44:04.016 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction test 2024-09-21T01:44:04.034 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction test succeeded. 2024-09-21T01:44:04.034 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction 2024-09-21T01:44:04.120 INFO:teuthology.orchestra.run.smithi105.stdout: Preparing : 1/1 2024-09-21T01:44:04.222 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T01:44:04.577 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T01:44:04.800 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T01:44:04.800 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:04.800 INFO:teuthology.orchestra.run.smithi105.stdout:Installed: 2024-09-21T01:44:04.801 INFO:teuthology.orchestra.run.smithi105.stdout: libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:44:04.801 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:04.801 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T01:44:04.966 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y install librados2 2024-09-21T01:44:05.540 INFO:teuthology.orchestra.run.smithi105.stdout:Last metadata expiration check: 0:01:00 ago on Sat 21 Sep 2024 01:43:05 AM UTC. 2024-09-21T01:44:05.630 INFO:teuthology.orchestra.run.smithi105.stdout:Package librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T01:44:05.671 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T01:44:05.672 INFO:teuthology.orchestra.run.smithi105.stdout:Nothing to do. 2024-09-21T01:44:05.672 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T01:44:05.751 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y install librbd1 2024-09-21T01:44:06.325 INFO:teuthology.orchestra.run.smithi105.stdout:Last metadata expiration check: 0:01:01 ago on Sat 21 Sep 2024 01:43:05 AM UTC. 2024-09-21T01:44:06.412 INFO:teuthology.orchestra.run.smithi105.stdout:Package librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T01:44:06.455 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T01:44:06.456 INFO:teuthology.orchestra.run.smithi105.stdout:Nothing to do. 2024-09-21T01:44:06.456 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T01:44:06.530 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y install python3-rados 2024-09-21T01:44:07.106 INFO:teuthology.orchestra.run.smithi105.stdout:Last metadata expiration check: 0:01:02 ago on Sat 21 Sep 2024 01:43:05 AM UTC. 2024-09-21T01:44:07.195 INFO:teuthology.orchestra.run.smithi105.stdout:Package python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T01:44:07.235 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T01:44:07.236 INFO:teuthology.orchestra.run.smithi105.stdout:Nothing to do. 2024-09-21T01:44:07.237 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T01:44:07.317 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y install python3-rgw 2024-09-21T01:44:07.897 INFO:teuthology.orchestra.run.smithi105.stdout:Last metadata expiration check: 0:01:02 ago on Sat 21 Sep 2024 01:43:05 AM UTC. 2024-09-21T01:44:07.987 INFO:teuthology.orchestra.run.smithi105.stdout:Package python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T01:44:08.028 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T01:44:08.029 INFO:teuthology.orchestra.run.smithi105.stdout:Nothing to do. 2024-09-21T01:44:08.029 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T01:44:08.098 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y install python3-cephfs 2024-09-21T01:44:08.672 INFO:teuthology.orchestra.run.smithi105.stdout:Last metadata expiration check: 0:01:03 ago on Sat 21 Sep 2024 01:43:05 AM UTC. 2024-09-21T01:44:08.761 INFO:teuthology.orchestra.run.smithi105.stdout:Package python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T01:44:08.803 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T01:44:08.804 INFO:teuthology.orchestra.run.smithi105.stdout:Nothing to do. 2024-09-21T01:44:08.804 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T01:44:08.884 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y install python3-rbd 2024-09-21T01:44:09.462 INFO:teuthology.orchestra.run.smithi105.stdout:Last metadata expiration check: 0:01:04 ago on Sat 21 Sep 2024 01:43:05 AM UTC. 2024-09-21T01:44:09.550 INFO:teuthology.orchestra.run.smithi105.stdout:Package python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T01:44:09.592 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T01:44:09.593 INFO:teuthology.orchestra.run.smithi105.stdout:Nothing to do. 2024-09-21T01:44:09.593 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T01:44:09.665 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y install rbd-fuse 2024-09-21T01:44:10.246 INFO:teuthology.orchestra.run.smithi105.stdout:Last metadata expiration check: 0:01:05 ago on Sat 21 Sep 2024 01:43:05 AM UTC. 2024-09-21T01:44:10.377 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T01:44:10.378 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:44:10.378 INFO:teuthology.orchestra.run.smithi105.stdout: Package Architecture Version Repository Size 2024-09-21T01:44:10.378 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:44:10.378 INFO:teuthology.orchestra.run.smithi105.stdout:Installing: 2024-09-21T01:44:10.379 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-fuse x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 87 k 2024-09-21T01:44:10.379 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:10.379 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction Summary 2024-09-21T01:44:10.379 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:44:10.379 INFO:teuthology.orchestra.run.smithi105.stdout:Install 1 Package 2024-09-21T01:44:10.379 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:10.379 INFO:teuthology.orchestra.run.smithi105.stdout:Total download size: 87 k 2024-09-21T01:44:10.379 INFO:teuthology.orchestra.run.smithi105.stdout:Installed size: 227 k 2024-09-21T01:44:10.379 INFO:teuthology.orchestra.run.smithi105.stdout:Downloading Packages: 2024-09-21T01:44:10.581 INFO:teuthology.orchestra.run.smithi105.stdout:rbd-fuse-18.2.4-871.g6a1bdf85.el9.x86_64.rpm 434 kB/s | 87 kB 00:00 2024-09-21T01:44:10.582 INFO:teuthology.orchestra.run.smithi105.stdout:-------------------------------------------------------------------------------- 2024-09-21T01:44:10.582 INFO:teuthology.orchestra.run.smithi105.stdout:Total 430 kB/s | 87 kB 00:00 2024-09-21T01:44:10.582 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction check 2024-09-21T01:44:10.588 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction check succeeded. 2024-09-21T01:44:10.589 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction test 2024-09-21T01:44:10.644 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction test succeeded. 2024-09-21T01:44:10.644 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction 2024-09-21T01:44:10.783 INFO:teuthology.orchestra.run.smithi105.stdout: Preparing : 1/1 2024-09-21T01:44:10.884 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T01:44:11.249 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T01:44:11.494 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T01:44:11.494 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:11.494 INFO:teuthology.orchestra.run.smithi105.stdout:Installed: 2024-09-21T01:44:11.495 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:44:11.495 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:11.495 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T01:44:11.722 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y install rbd-mirror 2024-09-21T01:44:12.308 INFO:teuthology.orchestra.run.smithi105.stdout:Last metadata expiration check: 0:01:07 ago on Sat 21 Sep 2024 01:43:05 AM UTC. 2024-09-21T01:44:12.440 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T01:44:12.441 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:44:12.441 INFO:teuthology.orchestra.run.smithi105.stdout: Package Arch Version Repo Size 2024-09-21T01:44:12.441 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:44:12.441 INFO:teuthology.orchestra.run.smithi105.stdout:Installing: 2024-09-21T01:44:12.441 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-mirror x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 3.0 M 2024-09-21T01:44:12.441 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:12.441 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction Summary 2024-09-21T01:44:12.442 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:44:12.442 INFO:teuthology.orchestra.run.smithi105.stdout:Install 1 Package 2024-09-21T01:44:12.442 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:12.442 INFO:teuthology.orchestra.run.smithi105.stdout:Total download size: 3.0 M 2024-09-21T01:44:12.442 INFO:teuthology.orchestra.run.smithi105.stdout:Installed size: 12 M 2024-09-21T01:44:12.442 INFO:teuthology.orchestra.run.smithi105.stdout:Downloading Packages: 2024-09-21T01:44:12.785 INFO:teuthology.orchestra.run.smithi105.stdout:rbd-mirror-18.2.4-871.g6a1bdf85.el9.x86_64.rpm 8.7 MB/s | 3.0 MB 00:00 2024-09-21T01:44:12.785 INFO:teuthology.orchestra.run.smithi105.stdout:-------------------------------------------------------------------------------- 2024-09-21T01:44:12.786 INFO:teuthology.orchestra.run.smithi105.stdout:Total 8.7 MB/s | 3.0 MB 00:00 2024-09-21T01:44:12.786 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction check 2024-09-21T01:44:12.795 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction check succeeded. 2024-09-21T01:44:12.795 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction test 2024-09-21T01:44:12.869 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction test succeeded. 2024-09-21T01:44:12.869 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction 2024-09-21T01:44:13.279 INFO:teuthology.orchestra.run.smithi105.stdout: Preparing : 1/1 2024-09-21T01:44:13.344 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T01:44:13.368 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T01:44:13.368 INFO:teuthology.orchestra.run.smithi105.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T01:44:13.368 INFO:teuthology.orchestra.run.smithi105.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-21T01:44:13.368 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T01:44:13.368 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T01:44:13.369 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:14.401 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T01:44:14.402 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:14.402 INFO:teuthology.orchestra.run.smithi105.stdout:Installed: 2024-09-21T01:44:14.402 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:44:14.402 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:14.402 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T01:44:14.664 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y install rbd-nbd 2024-09-21T01:44:15.258 INFO:teuthology.orchestra.run.smithi105.stdout:Last metadata expiration check: 0:01:10 ago on Sat 21 Sep 2024 01:43:05 AM UTC. 2024-09-21T01:44:15.391 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T01:44:15.392 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:44:15.392 INFO:teuthology.orchestra.run.smithi105.stdout: Package Architecture Version Repository Size 2024-09-21T01:44:15.392 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:44:15.392 INFO:teuthology.orchestra.run.smithi105.stdout:Installing: 2024-09-21T01:44:15.392 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-nbd x86_64 2:18.2.4-871.g6a1bdf85.el9 ceph 172 k 2024-09-21T01:44:15.392 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:15.392 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction Summary 2024-09-21T01:44:15.392 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:44:15.392 INFO:teuthology.orchestra.run.smithi105.stdout:Install 1 Package 2024-09-21T01:44:15.392 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:15.392 INFO:teuthology.orchestra.run.smithi105.stdout:Total download size: 172 k 2024-09-21T01:44:15.392 INFO:teuthology.orchestra.run.smithi105.stdout:Installed size: 494 k 2024-09-21T01:44:15.393 INFO:teuthology.orchestra.run.smithi105.stdout:Downloading Packages: 2024-09-21T01:44:15.621 INFO:teuthology.orchestra.run.smithi105.stdout:rbd-nbd-18.2.4-871.g6a1bdf85.el9.x86_64.rpm 753 kB/s | 172 kB 00:00 2024-09-21T01:44:15.622 INFO:teuthology.orchestra.run.smithi105.stdout:-------------------------------------------------------------------------------- 2024-09-21T01:44:15.622 INFO:teuthology.orchestra.run.smithi105.stdout:Total 746 kB/s | 172 kB 00:00 2024-09-21T01:44:15.623 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction check 2024-09-21T01:44:15.631 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction check succeeded. 2024-09-21T01:44:15.631 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction test 2024-09-21T01:44:15.691 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction test succeeded. 2024-09-21T01:44:15.691 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction 2024-09-21T01:44:15.850 INFO:teuthology.orchestra.run.smithi105.stdout: Preparing : 1/1 2024-09-21T01:44:15.962 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T01:44:16.440 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T01:44:16.694 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 1/1 2024-09-21T01:44:16.694 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:16.694 INFO:teuthology.orchestra.run.smithi105.stdout:Installed: 2024-09-21T01:44:16.695 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 2024-09-21T01:44:16.695 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:16.695 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T01:44:16.914 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y install librados2 2024-09-21T01:44:17.493 INFO:teuthology.orchestra.run.smithi105.stdout:Last metadata expiration check: 0:01:12 ago on Sat 21 Sep 2024 01:43:05 AM UTC. 2024-09-21T01:44:17.583 INFO:teuthology.orchestra.run.smithi105.stdout:Package librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 is already installed. 2024-09-21T01:44:17.624 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T01:44:17.625 INFO:teuthology.orchestra.run.smithi105.stdout:Nothing to do. 2024-09-21T01:44:17.625 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T01:44:17.706 DEBUG:teuthology.parallel:result is None 2024-09-21T01:44:17.707 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T01:44:17.707 INFO:teuthology.packaging:ref: None 2024-09-21T01:44:17.707 INFO:teuthology.packaging:tag: None 2024-09-21T01:44:17.707 INFO:teuthology.packaging:branch: reef 2024-09-21T01:44:17.707 INFO:teuthology.packaging:sha1: 1843221895adccfb3b692060251dd6a0183bc410 2024-09-21T01:44:17.707 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-21T01:44:17.931 DEBUG:teuthology.orchestra.run.smithi105:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-21T01:44:17.952 INFO:teuthology.orchestra.run.smithi105.stdout:18.2.4-871.g6a1bdf85.el9 2024-09-21T01:44:17.952 INFO:teuthology.packaging:The installed version of ceph is 18.2.4-871.g6a1bdf85.el9 2024-09-21T01:44:17.953 INFO:teuthology.task.install:The correct ceph version 18.2.4-871.g6a1bdf85 is installed. 2024-09-21T01:44:17.954 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-21T01:44:17.954 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:44:17.954 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-21T01:44:18.017 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-21T01:44:18.017 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:44:18.017 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/usr/bin/daemon-helper 2024-09-21T01:44:18.081 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-21T01:44:18.146 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-21T01:44:18.146 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:44:18.146 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-21T01:44:18.210 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-21T01:44:18.274 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-21T01:44:18.274 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:44:18.274 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/usr/bin/stdin-killer 2024-09-21T01:44:18.338 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-21T01:44:18.403 INFO:teuthology.run_tasks:Running task print... 2024-09-21T01:44:18.411 INFO:teuthology.task.print:**** done installing reef 2024-09-21T01:44:18.411 INFO:teuthology.run_tasks:Running task ceph... 2024-09-21T01:44:18.560 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2024-09-21T01:44:18.560 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod 777 /var/log/ceph 2024-09-21T01:44:18.584 INFO:tasks.ceph:Disabling ceph logrotate... 2024-09-21T01:44:18.584 DEBUG:teuthology.orchestra.run.smithi105:> sudo rm -f -- /etc/logrotate.d/ceph 2024-09-21T01:44:18.648 INFO:tasks.ceph:Creating extra log directories... 2024-09-21T01:44:18.648 DEBUG:teuthology.orchestra.run.smithi105:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2024-09-21T01:44:18.719 INFO:tasks.ceph:Creating ceph cluster ceph... 2024-09-21T01:44:18.719 INFO:tasks.ceph:config {'conf': {'global': {'mon warn on pool no app': False, 'ms bind msgr2': False, 'bluestore warn on legacy statfs': False, 'bluestore warn on no per pool omap': False, 'mon pg warn min per osd': 0}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon down mkfs grace': 300, 'mon warn on osd down out interval zero': False}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['overall HEALTH_', '\\(FS_', '\\(MDS_', '\\(OSD_', '\\(MON_DOWN\\)', '\\(CACHE_POOL_', '\\(POOL_', '\\(MGR_DOWN\\)', '\\(PG_', '\\(SMALLER_PGP_NUM\\)', 'Monitor daemon marked osd', 'Behind on trimming', 'Manager daemon', '\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'fs.*is degraded', 'filesystem is degraded', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'filesystem is offline', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'insufficient standby MDS daemons available', 'MDS_UP_LESS_THAN_MAX', 'online, but wants', 'filesystem is online with fewer MDS than max_mds', 'POOL_APP_NOT_ENABLED', 'do not have an application enabled', 'overall HEALTH_', 'Replacing daemon', 'deprecated feature inline_data', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding', 'PG_AVAILABILITY', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'scrub mismatch', 'ScrubResult', 'wrongly marked', '\\(POOL_APP_NOT_ENABLED\\)', '\\(SLOW_OPS\\)', 'overall HEALTH_', '\\(MON_MSGR2_NOT_ENABLED\\)', 'slow request'], 'cpu_profile': set(), 'cluster': 'ceph', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2024-09-21T01:44:18.720 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/archive/pdonnell-2024-09-20_23:48:22-fs-wip-pdonnell-testing-20240920.212106-debug-distro-default-smithi/7913395', 'branch': 'wip-pdonnell-testing-20240920.212106-debug', 'description': 'fs/upgrade/nofs/{bluestore-bitmap centos_9.stream conf/{client mds mgr mon osd} no-mds-cluster overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn pg_health} tasks/{0-from/reef 1-upgrade}}', 'email': 'pdonnell@redhat.com', 'first_in_suite': False, 'job_id': '7913395', 'kernel': {'kdb': 1, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'smithi', 'meta': [{'desc': 'install ceph/reef latest\n'}], 'name': 'pdonnell-2024-09-20_23:48:22-fs-wip-pdonnell-testing-20240920.212106-debug-distro-default-smithi', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 4, 'size': 10}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'wip-pdonnell-testing-20240920.212106-debug'}, 'ceph': {'cluster-conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 2, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mgr': {'client mount timeout': 30, 'debug client': 20, 'debug mgr': 20, 'debug ms': 1, 'mon warn on pool no app': False}, 'mon': {'mon op complaint time': 120}, 'osd': {'osd op complaint time': 180}}, 'conf': {'global': {'bluestore warn on legacy statfs': False, 'bluestore warn on no per pool omap': False, 'mon pg warn min per osd': 0}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon down mkfs grace': 300, 'mon warn on osd down out interval zero': False}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'fs.*is degraded', 'filesystem is degraded', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'filesystem is offline', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'insufficient standby MDS daemons available', 'MDS_UP_LESS_THAN_MAX', 'online, but wants', 'filesystem is online with fewer MDS than max_mds', 'POOL_APP_NOT_ENABLED', 'do not have an application enabled', 'overall HEALTH_', 'Replacing daemon', 'deprecated feature inline_data', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding', 'PG_AVAILABILITY', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'scrub mismatch', 'ScrubResult', 'wrongly marked', '\\(POOL_APP_NOT_ENABLED\\)', '\\(SLOW_OPS\\)', 'overall HEALTH_', '\\(MON_MSGR2_NOT_ENABLED\\)', 'slow request'], 'sha1': '1843221895adccfb3b692060251dd6a0183bc410'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': '1843221895adccfb3b692060251dd6a0183bc410'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-pdonnell-testing-20240920.212106-debug', 'sha1': '1843221895adccfb3b692060251dd6a0183bc410'}}, 'owner': 'scheduled_pdonnell@teuthology', 'priority': 90, 'repo': 'https://git.ceph.com/ceph-ci.git', 'roles': [['mon.a', 'mon.b', 'mon.c', 'mgr.x', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3']], 'seed': 7864, 'sha1': '1843221895adccfb3b692060251dd6a0183bc410', 'sleep_before_teardown': 0, 'subset': '67/512', 'suite': 'fs', 'suite_branch': 'wip-pdonnell-testing-20240920.212106-debug', 'suite_path': '/home/teuthworker/src/git.ceph.com_ceph-c_1843221895adccfb3b692060251dd6a0183bc410/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://git.ceph.com/ceph-ci.git', 'suite_sha1': '1843221895adccfb3b692060251dd6a0183bc410', 'targets': {'smithi105.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOfZSKdnL6ONyv5UGvKyQeUbyh7h+S0cxRlXb6GX6LKEL3HqXfi8wMvJ901BrKQ1NAm6rq8A5tYAvXo2rq8MnH8='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'kdb': 1, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'branch': 'reef', 'exclude_packages': ['librados3', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'cephadm', 'ceph-volume'], 'extra_packages': ['librados2'], 'flavor': 'default', 'sha1': '1843221895adccfb3b692060251dd6a0183bc410'}}, {'print': '**** done installing reef'}, {'ceph': {'conf': {'global': {'mon warn on pool no app': False, 'ms bind msgr2': False, 'bluestore warn on legacy statfs': False, 'bluestore warn on no per pool omap': False, 'mon pg warn min per osd': 0}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon down mkfs grace': 300, 'mon warn on osd down out interval zero': False}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'log-ignorelist': ['overall HEALTH_', '\\(FS_', '\\(MDS_', '\\(OSD_', '\\(MON_DOWN\\)', '\\(CACHE_POOL_', '\\(POOL_', '\\(MGR_DOWN\\)', '\\(PG_', '\\(SMALLER_PGP_NUM\\)', 'Monitor daemon marked osd', 'Behind on trimming', 'Manager daemon', '\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'fs.*is degraded', 'filesystem is degraded', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'filesystem is offline', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'insufficient standby MDS daemons available', 'MDS_UP_LESS_THAN_MAX', 'online, but wants', 'filesystem is online with fewer MDS than max_mds', 'POOL_APP_NOT_ENABLED', 'do not have an application enabled', 'overall HEALTH_', 'Replacing daemon', 'deprecated feature inline_data', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding', 'PG_AVAILABILITY', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'scrub mismatch', 'ScrubResult', 'wrongly marked', '\\(POOL_APP_NOT_ENABLED\\)', '\\(SLOW_OPS\\)', 'overall HEALTH_', '\\(MON_MSGR2_NOT_ENABLED\\)', 'slow request'], 'cluster-conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 2, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mgr': {'client mount timeout': 30, 'debug client': 20, 'debug mgr': 20, 'debug ms': 1, 'mon warn on pool no app': False}, 'mon': {'mon op complaint time': 120}, 'osd': {'osd op complaint time': 180}}, 'flavor': 'default', 'fs': 'xfs', 'sha1': '1843221895adccfb3b692060251dd6a0183bc410', 'cluster': 'ceph'}}, {'exec': {'osd.0': ['ceph osd set-require-min-compat-client reef']}}, {'print': '**** done ceph'}, {'print': '*** upgrading, no cephfs present'}, {'exec': {'mon.a': ['ceph fs dump']}}, {'install.upgrade': {'mon.a': None}}, {'print': '**** done install.upgrade'}, {'ceph.restart': {'daemons': ['mon.*', 'mgr.*'], 'mon-health-to-clog': False, 'wait-for-healthy': False}}, {'ceph.healthy': None}, {'ceph.restart': {'daemons': ['osd.*'], 'wait-for-healthy': False, 'wait-for-osds-up': True}}, {'exec': {'mon.a': ['ceph versions', 'ceph osd dump -f json-pretty', 'ceph fs dump', 'ceph osd require-osd-release squid', 'for f in `ceph osd pool ls` ; do ceph osd pool set $f pg_autoscale_mode off ; done']}}, {'ceph.healthy': None}, {'print': '**** done ceph.restart'}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': '3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a', 'timestamp': '2024-09-20_23:48:22', 'tube': 'smithi', 'user': 'pdonnell', 'verbose': False, 'worker_log': '/home/teuthworker/archive/worker_logs/dispatcher.smithi.2902'} 2024-09-21T01:44:18.720 DEBUG:teuthology.orchestra.run.smithi105:> install -d -m0755 -- /home/ubuntu/cephtest/ceph.data 2024-09-21T01:44:18.768 DEBUG:teuthology.orchestra.run.smithi105:> sudo install -d -m0777 -- /var/run/ceph 2024-09-21T01:44:18.833 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:44:18.833 DEBUG:teuthology.orchestra.run.smithi105:> dd if=/scratch_devs of=/dev/stdout 2024-09-21T01:44:18.889 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-21T01:44:18.889 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_1 2024-09-21T01:44:18.944 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-21T01:44:18.944 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T01:44:18.944 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-09-21T01:44:18.944 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T01:44:18.944 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T01:44:18.944 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-09-21 01:44:13.609159891 +0000 2024-09-21T01:44:18.944 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-09-21 01:42:26.123135846 +0000 2024-09-21T01:44:18.945 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-09-21 01:42:26.123135846 +0000 2024-09-21T01:44:18.945 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: 2024-09-21 01:42:26.123135846 +0000 2024-09-21T01:44:18.945 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-21T01:44:19.006 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-09-21T01:44:19.007 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-09-21T01:44:19.007 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000179055 s, 2.9 MB/s 2024-09-21T01:44:19.008 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-21T01:44:19.064 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_2 2024-09-21T01:44:19.119 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-21T01:44:19.119 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T01:44:19.119 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-09-21T01:44:19.119 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T01:44:19.119 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T01:44:19.119 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-09-21 01:44:13.609159891 +0000 2024-09-21T01:44:19.119 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-09-21 01:42:26.442129789 +0000 2024-09-21T01:44:19.119 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-09-21 01:42:26.442129789 +0000 2024-09-21T01:44:19.120 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: 2024-09-21 01:42:26.442129789 +0000 2024-09-21T01:44:19.120 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-21T01:44:19.181 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-09-21T01:44:19.182 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-09-21T01:44:19.182 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000219356 s, 2.3 MB/s 2024-09-21T01:44:19.183 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-21T01:44:19.239 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_3 2024-09-21T01:44:19.294 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-21T01:44:19.294 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T01:44:19.294 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-09-21T01:44:19.294 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T01:44:19.294 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T01:44:19.294 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-09-21 01:44:13.609159891 +0000 2024-09-21T01:44:19.294 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-09-21 01:42:26.769123580 +0000 2024-09-21T01:44:19.294 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-09-21 01:42:26.769123580 +0000 2024-09-21T01:44:19.295 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: 2024-09-21 01:42:26.769123580 +0000 2024-09-21T01:44:19.295 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-21T01:44:19.356 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-09-21T01:44:19.357 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-09-21T01:44:19.357 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000178332 s, 2.9 MB/s 2024-09-21T01:44:19.358 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-21T01:44:19.414 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_4 2024-09-21T01:44:19.469 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-21T01:44:19.469 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T01:44:19.470 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-09-21T01:44:19.470 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T01:44:19.470 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T01:44:19.470 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-09-21 01:44:13.610159872 +0000 2024-09-21T01:44:19.470 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-09-21 01:42:27.073117808 +0000 2024-09-21T01:44:19.470 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-09-21 01:42:27.073117808 +0000 2024-09-21T01:44:19.470 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: 2024-09-21 01:42:27.073117808 +0000 2024-09-21T01:44:19.470 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-21T01:44:19.533 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-09-21T01:44:19.533 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-09-21T01:44:19.533 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000253029 s, 2.0 MB/s 2024-09-21T01:44:19.534 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-21T01:44:19.592 INFO:tasks.ceph:osd dev map: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-09-21T01:44:19.592 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@smithi105.front.sepia.ceph.com'): {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'}} 2024-09-21T01:44:19.592 INFO:tasks.ceph:Generating config... 2024-09-21T01:44:19.594 INFO:tasks.ceph:[global] mon warn on pool no app = False 2024-09-21T01:44:19.594 INFO:tasks.ceph:[global] ms bind msgr2 = False 2024-09-21T01:44:19.594 INFO:tasks.ceph:[global] bluestore warn on legacy statfs = False 2024-09-21T01:44:19.594 INFO:tasks.ceph:[global] bluestore warn on no per pool omap = False 2024-09-21T01:44:19.594 INFO:tasks.ceph:[global] mon pg warn min per osd = 0 2024-09-21T01:44:19.595 INFO:tasks.ceph:[mgr] debug mgr = 20 2024-09-21T01:44:19.595 INFO:tasks.ceph:[mgr] debug ms = 1 2024-09-21T01:44:19.595 INFO:tasks.ceph:[mon] debug mon = 20 2024-09-21T01:44:19.595 INFO:tasks.ceph:[mon] debug ms = 1 2024-09-21T01:44:19.595 INFO:tasks.ceph:[mon] debug paxos = 20 2024-09-21T01:44:19.595 INFO:tasks.ceph:[mon] mon down mkfs grace = 300 2024-09-21T01:44:19.595 INFO:tasks.ceph:[mon] mon warn on osd down out interval zero = False 2024-09-21T01:44:19.595 INFO:tasks.ceph:[osd] bdev async discard = True 2024-09-21T01:44:19.596 INFO:tasks.ceph:[osd] bdev enable discard = True 2024-09-21T01:44:19.596 INFO:tasks.ceph:[osd] bluestore allocator = bitmap 2024-09-21T01:44:19.596 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2024-09-21T01:44:19.596 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2024-09-21T01:44:19.596 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2024-09-21T01:44:19.596 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2024-09-21T01:44:19.596 INFO:tasks.ceph:[osd] debug ms = 1 2024-09-21T01:44:19.596 INFO:tasks.ceph:[osd] debug osd = 20 2024-09-21T01:44:19.596 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2024-09-21T01:44:19.596 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2024-09-21T01:44:19.596 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2024-09-21T01:44:19.596 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2024-09-21T01:44:19.596 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2024-09-21T01:44:19.596 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2024-09-21T01:44:19.597 INFO:tasks.ceph:Setting up mon.a... 2024-09-21T01:44:19.597 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/ceph.keyring 2024-09-21T01:44:19.684 INFO:teuthology.orchestra.run.smithi105.stdout:creating /etc/ceph/ceph.keyring 2024-09-21T01:44:19.687 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/ceph.keyring 2024-09-21T01:44:19.728 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod 0644 /etc/ceph/ceph.keyring 2024-09-21T01:44:19.792 DEBUG:tasks.ceph:Ceph mon addresses: [('mon.a', '172.21.15.105'), ('mon.b', '[v2:172.21.15.105:3301,v1:172.21.15.105:6790]'), ('mon.c', '[v2:172.21.15.105:3302,v1:172.21.15.105:6791]')] 2024-09-21T01:44:19.792 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '172.21.15.105,[v2:172.21.15.105:3301,v1:172.21.15.105:6790],[v2:172.21.15.105:3302,v1:172.21.15.105:6791]', 'mon warn on pool no app': False, 'ms bind msgr2': False, 'bluestore warn on legacy statfs': False, 'bluestore warn on no per pool omap': False, 'mon pg warn min per osd': 0}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false', 'mon down mkfs grace': 300, 'mon warn on osd down out interval zero': False}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok'}, 'mon.a': {}, 'mon.b': {}, 'mon.c': {}} 2024-09-21T01:44:19.793 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:44:19.793 DEBUG:teuthology.orchestra.run.smithi105:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2024-09-21T01:44:19.847 DEBUG:teuthology.orchestra.run.smithi105:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 172.21.15.105 --addv b '[v2:172.21.15.105:3301,v1:172.21.15.105:6790]' --addv c '[v2:172.21.15.105:3302,v1:172.21.15.105:6791]' --print /home/ubuntu/cephtest/ceph.monmap 2024-09-21T01:44:19.932 INFO:teuthology.orchestra.run.smithi105.stdout:monmaptool: monmap file /home/ubuntu/cephtest/ceph.monmap 2024-09-21T01:44:19.932 INFO:teuthology.orchestra.run.smithi105.stdout:monmaptool: generated fsid 355399d3-6490-4ad0-ac52-d793648ff276 2024-09-21T01:44:19.932 INFO:teuthology.orchestra.run.smithi105.stdout:setting min_mon_release = pacific 2024-09-21T01:44:19.932 INFO:teuthology.orchestra.run.smithi105.stdout:epoch 0 2024-09-21T01:44:19.932 INFO:teuthology.orchestra.run.smithi105.stdout:fsid 355399d3-6490-4ad0-ac52-d793648ff276 2024-09-21T01:44:19.932 INFO:teuthology.orchestra.run.smithi105.stdout:last_changed 2024-09-21T01:44:19.932832+0000 2024-09-21T01:44:19.932 INFO:teuthology.orchestra.run.smithi105.stdout:created 2024-09-21T01:44:19.932832+0000 2024-09-21T01:44:19.932 INFO:teuthology.orchestra.run.smithi105.stdout:min_mon_release 16 (pacific) 2024-09-21T01:44:19.932 INFO:teuthology.orchestra.run.smithi105.stdout:election_strategy: 1 2024-09-21T01:44:19.933 INFO:teuthology.orchestra.run.smithi105.stdout:0: [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon.a 2024-09-21T01:44:19.933 INFO:teuthology.orchestra.run.smithi105.stdout:1: [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] mon.b 2024-09-21T01:44:19.933 INFO:teuthology.orchestra.run.smithi105.stdout:2: [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] mon.c 2024-09-21T01:44:19.933 INFO:teuthology.orchestra.run.smithi105.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/ceph.monmap (3 monitors) 2024-09-21T01:44:19.934 DEBUG:teuthology.orchestra.run.smithi105:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2024-09-21T01:44:19.948 INFO:tasks.ceph:Writing /etc/ceph/ceph.conf for FSID 355399d3-6490-4ad0-ac52-d793648ff276... 2024-09-21T01:44:19.949 DEBUG:teuthology.orchestra.run.smithi105:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf > /dev/null 2024-09-21T01:44:20.030 INFO:teuthology.orchestra.run.smithi105.stdout:[global] 2024-09-21T01:44:20.030 INFO:teuthology.orchestra.run.smithi105.stdout: chdir = "" 2024-09-21T01:44:20.031 INFO:teuthology.orchestra.run.smithi105.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2024-09-21T01:44:20.031 INFO:teuthology.orchestra.run.smithi105.stdout: auth supported = cephx 2024-09-21T01:44:20.031 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.031 INFO:teuthology.orchestra.run.smithi105.stdout: filestore xattr use omap = true 2024-09-21T01:44:20.031 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.031 INFO:teuthology.orchestra.run.smithi105.stdout: mon clock drift allowed = 1.000 2024-09-21T01:44:20.031 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.031 INFO:teuthology.orchestra.run.smithi105.stdout: osd crush chooseleaf type = 0 2024-09-21T01:44:20.031 INFO:teuthology.orchestra.run.smithi105.stdout: auth debug = true 2024-09-21T01:44:20.031 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.031 INFO:teuthology.orchestra.run.smithi105.stdout: ms die on old message = true 2024-09-21T01:44:20.031 INFO:teuthology.orchestra.run.smithi105.stdout: ms die on bug = true 2024-09-21T01:44:20.031 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.031 INFO:teuthology.orchestra.run.smithi105.stdout: mon max pg per osd = 10000 # >= luminous 2024-09-21T01:44:20.032 INFO:teuthology.orchestra.run.smithi105.stdout: mon pg warn max object skew = 0 2024-09-21T01:44:20.032 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.032 INFO:teuthology.orchestra.run.smithi105.stdout: # disable pg_autoscaler by default for new pools 2024-09-21T01:44:20.032 INFO:teuthology.orchestra.run.smithi105.stdout: osd_pool_default_pg_autoscale_mode = off 2024-09-21T01:44:20.032 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.032 INFO:teuthology.orchestra.run.smithi105.stdout: osd pool default size = 2 2024-09-21T01:44:20.032 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.032 INFO:teuthology.orchestra.run.smithi105.stdout: mon osd allow primary affinity = true 2024-09-21T01:44:20.032 INFO:teuthology.orchestra.run.smithi105.stdout: mon osd allow pg remap = true 2024-09-21T01:44:20.032 INFO:teuthology.orchestra.run.smithi105.stdout: mon warn on legacy crush tunables = false 2024-09-21T01:44:20.032 INFO:teuthology.orchestra.run.smithi105.stdout: mon warn on crush straw calc version zero = false 2024-09-21T01:44:20.032 INFO:teuthology.orchestra.run.smithi105.stdout: mon warn on no sortbitwise = false 2024-09-21T01:44:20.032 INFO:teuthology.orchestra.run.smithi105.stdout: mon warn on osd down out interval zero = false 2024-09-21T01:44:20.032 INFO:teuthology.orchestra.run.smithi105.stdout: mon warn on too few osds = false 2024-09-21T01:44:20.032 INFO:teuthology.orchestra.run.smithi105.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2024-09-21T01:44:20.033 INFO:teuthology.orchestra.run.smithi105.stdout: mon_warn_on_pool_no_redundancy = false 2024-09-21T01:44:20.033 INFO:teuthology.orchestra.run.smithi105.stdout: mon_allow_pool_size_one = true 2024-09-21T01:44:20.033 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.033 INFO:teuthology.orchestra.run.smithi105.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2024-09-21T01:44:20.033 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.033 INFO:teuthology.orchestra.run.smithi105.stdout: osd default data pool replay window = 5 2024-09-21T01:44:20.033 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.033 INFO:teuthology.orchestra.run.smithi105.stdout: mon allow pool delete = true 2024-09-21T01:44:20.033 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.033 INFO:teuthology.orchestra.run.smithi105.stdout: mon cluster log file level = debug 2024-09-21T01:44:20.033 INFO:teuthology.orchestra.run.smithi105.stdout: debug asserts on shutdown = true 2024-09-21T01:44:20.033 INFO:teuthology.orchestra.run.smithi105.stdout: mon health detail to clog = false 2024-09-21T01:44:20.033 INFO:teuthology.orchestra.run.smithi105.stdout: mon host = "172.21.15.105,[v2:172.21.15.105:3301,v1:172.21.15.105:6790],[v2:172.21.15.105:3302,v1:172.21.15.105:6791]" 2024-09-21T01:44:20.033 INFO:teuthology.orchestra.run.smithi105.stdout: mon warn on pool no app = False 2024-09-21T01:44:20.034 INFO:teuthology.orchestra.run.smithi105.stdout: ms bind msgr2 = False 2024-09-21T01:44:20.034 INFO:teuthology.orchestra.run.smithi105.stdout: bluestore warn on legacy statfs = False 2024-09-21T01:44:20.034 INFO:teuthology.orchestra.run.smithi105.stdout: bluestore warn on no per pool omap = False 2024-09-21T01:44:20.034 INFO:teuthology.orchestra.run.smithi105.stdout: mon pg warn min per osd = 0 2024-09-21T01:44:20.034 INFO:teuthology.orchestra.run.smithi105.stdout: fsid = 355399d3-6490-4ad0-ac52-d793648ff276 2024-09-21T01:44:20.034 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.034 INFO:teuthology.orchestra.run.smithi105.stdout:[osd] 2024-09-21T01:44:20.034 INFO:teuthology.orchestra.run.smithi105.stdout: osd journal size = 100 2024-09-21T01:44:20.034 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.034 INFO:teuthology.orchestra.run.smithi105.stdout: osd scrub load threshold = 5.0 2024-09-21T01:44:20.034 INFO:teuthology.orchestra.run.smithi105.stdout: osd scrub max interval = 600 2024-09-21T01:44:20.034 INFO:teuthology.orchestra.run.smithi105.stdout: osd mclock profile = high_recovery_ops 2024-09-21T01:44:20.034 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.034 INFO:teuthology.orchestra.run.smithi105.stdout: osd recover clone overlap = true 2024-09-21T01:44:20.035 INFO:teuthology.orchestra.run.smithi105.stdout: osd recovery max chunk = 1048576 2024-09-21T01:44:20.035 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.035 INFO:teuthology.orchestra.run.smithi105.stdout: osd debug shutdown = true 2024-09-21T01:44:20.035 INFO:teuthology.orchestra.run.smithi105.stdout: osd debug op order = true 2024-09-21T01:44:20.035 INFO:teuthology.orchestra.run.smithi105.stdout: osd debug verify stray on activate = true 2024-09-21T01:44:20.035 INFO:teuthology.orchestra.run.smithi105.stdout: osd debug trim objects = true 2024-09-21T01:44:20.035 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.035 INFO:teuthology.orchestra.run.smithi105.stdout: osd open classes on start = true 2024-09-21T01:44:20.035 INFO:teuthology.orchestra.run.smithi105.stdout: osd debug pg log writeout = true 2024-09-21T01:44:20.035 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.035 INFO:teuthology.orchestra.run.smithi105.stdout: osd deep scrub update digest min age = 30 2024-09-21T01:44:20.035 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.035 INFO:teuthology.orchestra.run.smithi105.stdout: osd map max advance = 10 2024-09-21T01:44:20.035 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.035 INFO:teuthology.orchestra.run.smithi105.stdout: journal zero on create = true 2024-09-21T01:44:20.036 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.036 INFO:teuthology.orchestra.run.smithi105.stdout: filestore ondisk finisher threads = 3 2024-09-21T01:44:20.036 INFO:teuthology.orchestra.run.smithi105.stdout: filestore apply finisher threads = 3 2024-09-21T01:44:20.036 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.036 INFO:teuthology.orchestra.run.smithi105.stdout: bdev debug aio = true 2024-09-21T01:44:20.036 INFO:teuthology.orchestra.run.smithi105.stdout: osd debug misdirected ops = true 2024-09-21T01:44:20.036 INFO:teuthology.orchestra.run.smithi105.stdout: bdev async discard = True 2024-09-21T01:44:20.036 INFO:teuthology.orchestra.run.smithi105.stdout: bdev enable discard = True 2024-09-21T01:44:20.036 INFO:teuthology.orchestra.run.smithi105.stdout: bluestore allocator = bitmap 2024-09-21T01:44:20.036 INFO:teuthology.orchestra.run.smithi105.stdout: bluestore block size = 96636764160 2024-09-21T01:44:20.036 INFO:teuthology.orchestra.run.smithi105.stdout: bluestore fsck on mount = True 2024-09-21T01:44:20.036 INFO:teuthology.orchestra.run.smithi105.stdout: debug bluefs = 1/20 2024-09-21T01:44:20.036 INFO:teuthology.orchestra.run.smithi105.stdout: debug bluestore = 1/20 2024-09-21T01:44:20.036 INFO:teuthology.orchestra.run.smithi105.stdout: debug ms = 1 2024-09-21T01:44:20.036 INFO:teuthology.orchestra.run.smithi105.stdout: debug osd = 20 2024-09-21T01:44:20.037 INFO:teuthology.orchestra.run.smithi105.stdout: debug rocksdb = 4/10 2024-09-21T01:44:20.037 INFO:teuthology.orchestra.run.smithi105.stdout: mon osd backfillfull_ratio = 0.85 2024-09-21T01:44:20.037 INFO:teuthology.orchestra.run.smithi105.stdout: mon osd full ratio = 0.9 2024-09-21T01:44:20.037 INFO:teuthology.orchestra.run.smithi105.stdout: mon osd nearfull ratio = 0.8 2024-09-21T01:44:20.037 INFO:teuthology.orchestra.run.smithi105.stdout: osd failsafe full ratio = 0.95 2024-09-21T01:44:20.037 INFO:teuthology.orchestra.run.smithi105.stdout: osd objectstore = bluestore 2024-09-21T01:44:20.037 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.037 INFO:teuthology.orchestra.run.smithi105.stdout:[mgr] 2024-09-21T01:44:20.037 INFO:teuthology.orchestra.run.smithi105.stdout: debug ms = 1 2024-09-21T01:44:20.037 INFO:teuthology.orchestra.run.smithi105.stdout: debug mgr = 20 2024-09-21T01:44:20.037 INFO:teuthology.orchestra.run.smithi105.stdout: debug mon = 20 2024-09-21T01:44:20.037 INFO:teuthology.orchestra.run.smithi105.stdout: debug auth = 20 2024-09-21T01:44:20.037 INFO:teuthology.orchestra.run.smithi105.stdout: mon reweight min pgs per osd = 4 2024-09-21T01:44:20.037 INFO:teuthology.orchestra.run.smithi105.stdout: mon reweight min bytes per osd = 10 2024-09-21T01:44:20.037 INFO:teuthology.orchestra.run.smithi105.stdout: mgr/telemetry/nag = false 2024-09-21T01:44:20.038 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.038 INFO:teuthology.orchestra.run.smithi105.stdout:[mon] 2024-09-21T01:44:20.038 INFO:teuthology.orchestra.run.smithi105.stdout: debug ms = 1 2024-09-21T01:44:20.038 INFO:teuthology.orchestra.run.smithi105.stdout: debug mon = 20 2024-09-21T01:44:20.038 INFO:teuthology.orchestra.run.smithi105.stdout: debug paxos = 20 2024-09-21T01:44:20.038 INFO:teuthology.orchestra.run.smithi105.stdout: debug auth = 20 2024-09-21T01:44:20.038 INFO:teuthology.orchestra.run.smithi105.stdout: mon data avail warn = 5 2024-09-21T01:44:20.038 INFO:teuthology.orchestra.run.smithi105.stdout: mon mgr mkfs grace = 240 2024-09-21T01:44:20.038 INFO:teuthology.orchestra.run.smithi105.stdout: mon reweight min pgs per osd = 4 2024-09-21T01:44:20.038 INFO:teuthology.orchestra.run.smithi105.stdout: mon osd reporter subtree level = osd 2024-09-21T01:44:20.038 INFO:teuthology.orchestra.run.smithi105.stdout: mon osd prime pg temp = true 2024-09-21T01:44:20.038 INFO:teuthology.orchestra.run.smithi105.stdout: mon reweight min bytes per osd = 10 2024-09-21T01:44:20.038 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.038 INFO:teuthology.orchestra.run.smithi105.stdout: # rotate auth tickets quickly to exercise renewal paths 2024-09-21T01:44:20.039 INFO:teuthology.orchestra.run.smithi105.stdout: auth mon ticket ttl = 660 # 11m 2024-09-21T01:44:20.039 INFO:teuthology.orchestra.run.smithi105.stdout: auth service ticket ttl = 240 # 4m 2024-09-21T01:44:20.039 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.039 INFO:teuthology.orchestra.run.smithi105.stdout: # don't complain about insecure global_id in the test suite 2024-09-21T01:44:20.039 INFO:teuthology.orchestra.run.smithi105.stdout: mon_warn_on_insecure_global_id_reclaim = false 2024-09-21T01:44:20.039 INFO:teuthology.orchestra.run.smithi105.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2024-09-21T01:44:20.039 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.039 INFO:teuthology.orchestra.run.smithi105.stdout: # 1m isn't quite enough 2024-09-21T01:44:20.039 INFO:teuthology.orchestra.run.smithi105.stdout: mon_down_mkfs_grace = 2m 2024-09-21T01:44:20.039 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.039 INFO:teuthology.orchestra.run.smithi105.stdout: mon_warn_on_filestore_osds = false 2024-09-21T01:44:20.039 INFO:teuthology.orchestra.run.smithi105.stdout: mon down mkfs grace = 300 2024-09-21T01:44:20.039 INFO:teuthology.orchestra.run.smithi105.stdout: mon warn on osd down out interval zero = False 2024-09-21T01:44:20.039 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:20.040 INFO:teuthology.orchestra.run.smithi105.stdout:[client] 2024-09-21T01:44:20.040 INFO:teuthology.orchestra.run.smithi105.stdout: rgw cache enabled = true 2024-09-21T01:44:20.040 INFO:teuthology.orchestra.run.smithi105.stdout: rgw enable ops log = true 2024-09-21T01:44:20.040 INFO:teuthology.orchestra.run.smithi105.stdout: rgw enable usage log = true 2024-09-21T01:44:20.041 INFO:teuthology.orchestra.run.smithi105.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2024-09-21T01:44:20.041 INFO:teuthology.orchestra.run.smithi105.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2024-09-21T01:44:20.041 INFO:teuthology.orchestra.run.smithi105.stdout:[mon.a] 2024-09-21T01:44:20.042 INFO:teuthology.orchestra.run.smithi105.stdout:[mon.b] 2024-09-21T01:44:20.042 INFO:teuthology.orchestra.run.smithi105.stdout:[mon.c] 2024-09-21T01:44:20.042 INFO:tasks.ceph:Creating admin key on mon.a... 2024-09-21T01:44:20.043 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/ceph.keyring 2024-09-21T01:44:20.136 INFO:tasks.ceph:Copying monmap to all nodes... 2024-09-21T01:44:20.136 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:44:20.136 DEBUG:teuthology.orchestra.run.smithi105:> dd if=/etc/ceph/ceph.keyring of=/dev/stdout 2024-09-21T01:44:20.152 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:44:20.152 DEBUG:teuthology.orchestra.run.smithi105:> dd if=/home/ubuntu/cephtest/ceph.monmap of=/dev/stdout 2024-09-21T01:44:20.207 INFO:tasks.ceph:Sending monmap to node ubuntu@smithi105.front.sepia.ceph.com 2024-09-21T01:44:20.207 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:44:20.207 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/ceph/ceph.keyring 2024-09-21T01:44:20.207 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod 0644 /etc/ceph/ceph.keyring 2024-09-21T01:44:20.281 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:44:20.281 DEBUG:teuthology.orchestra.run.smithi105:> dd of=/home/ubuntu/cephtest/ceph.monmap 2024-09-21T01:44:20.336 INFO:tasks.ceph:Setting up mon nodes... 2024-09-21T01:44:20.336 INFO:tasks.ceph:Setting up mgr nodes... 2024-09-21T01:44:20.337 DEBUG:teuthology.orchestra.run.smithi105:> sudo mkdir -p /var/lib/ceph/mgr/ceph-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/ceph-x/keyring 2024-09-21T01:44:20.427 INFO:teuthology.orchestra.run.smithi105.stdout:creating /var/lib/ceph/mgr/ceph-x/keyring 2024-09-21T01:44:20.430 DEBUG:teuthology.orchestra.run.smithi105:> sudo mkdir -p /var/lib/ceph/mgr/ceph-y && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.y /var/lib/ceph/mgr/ceph-y/keyring 2024-09-21T01:44:20.479 INFO:teuthology.orchestra.run.smithi105.stdout:creating /var/lib/ceph/mgr/ceph-y/keyring 2024-09-21T01:44:20.482 INFO:tasks.ceph:Setting up mds nodes... 2024-09-21T01:44:20.482 INFO:tasks.ceph_client:Setting up client nodes... 2024-09-21T01:44:20.482 INFO:tasks.ceph:Running mkfs on osd nodes... 2024-09-21T01:44:20.482 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@smithi105.front.sepia.ceph.com'): {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'}} 2024-09-21T01:44:20.482 DEBUG:teuthology.orchestra.run.smithi105:> sudo mkdir -p /var/lib/ceph/osd/ceph-0 2024-09-21T01:44:20.506 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-09-21T01:44:20.507 INFO:tasks.ceph:role: osd.0 2024-09-21T01:44:20.507 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@smithi105.front.sepia.ceph.com 2024-09-21T01:44:20.507 DEBUG:teuthology.orchestra.run.smithi105:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2024-09-21T01:44:20.579 INFO:teuthology.orchestra.run.smithi105.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=5859072 blks 2024-09-21T01:44:20.579 INFO:teuthology.orchestra.run.smithi105.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-21T01:44:20.579 INFO:teuthology.orchestra.run.smithi105.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-21T01:44:20.580 INFO:teuthology.orchestra.run.smithi105.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-21T01:44:20.580 INFO:teuthology.orchestra.run.smithi105.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-21T01:44:20.580 INFO:teuthology.orchestra.run.smithi105.stdout: = sunit=0 swidth=0 blks 2024-09-21T01:44:20.580 INFO:teuthology.orchestra.run.smithi105.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-21T01:44:20.580 INFO:teuthology.orchestra.run.smithi105.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-21T01:44:20.580 INFO:teuthology.orchestra.run.smithi105.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-21T01:44:20.580 INFO:teuthology.orchestra.run.smithi105.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-21T01:44:21.074 INFO:teuthology.orchestra.run.smithi105.stdout:Discarding blocks...Done. 2024-09-21T01:44:21.076 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@smithi105.front.sepia.ceph.com -o noatime 2024-09-21T01:44:21.077 DEBUG:teuthology.orchestra.run.smithi105:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-0 2024-09-21T01:44:21.229 DEBUG:teuthology.orchestra.run.smithi105:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-0 2024-09-21T01:44:21.260 DEBUG:teuthology.orchestra.run.smithi105:> sudo mkdir -p /var/lib/ceph/osd/ceph-1 2024-09-21T01:44:21.326 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-09-21T01:44:21.327 INFO:tasks.ceph:role: osd.1 2024-09-21T01:44:21.327 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@smithi105.front.sepia.ceph.com 2024-09-21T01:44:21.327 DEBUG:teuthology.orchestra.run.smithi105:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2024-09-21T01:44:21.399 INFO:teuthology.orchestra.run.smithi105.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=5859072 blks 2024-09-21T01:44:21.399 INFO:teuthology.orchestra.run.smithi105.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-21T01:44:21.399 INFO:teuthology.orchestra.run.smithi105.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-21T01:44:21.399 INFO:teuthology.orchestra.run.smithi105.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-21T01:44:21.399 INFO:teuthology.orchestra.run.smithi105.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-21T01:44:21.400 INFO:teuthology.orchestra.run.smithi105.stdout: = sunit=0 swidth=0 blks 2024-09-21T01:44:21.400 INFO:teuthology.orchestra.run.smithi105.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-21T01:44:21.400 INFO:teuthology.orchestra.run.smithi105.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-21T01:44:21.400 INFO:teuthology.orchestra.run.smithi105.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-21T01:44:21.400 INFO:teuthology.orchestra.run.smithi105.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-21T01:44:21.859 INFO:teuthology.orchestra.run.smithi105.stdout:Discarding blocks...Done. 2024-09-21T01:44:21.861 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@smithi105.front.sepia.ceph.com -o noatime 2024-09-21T01:44:21.861 DEBUG:teuthology.orchestra.run.smithi105:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-1 2024-09-21T01:44:21.955 DEBUG:teuthology.orchestra.run.smithi105:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-1 2024-09-21T01:44:21.983 DEBUG:teuthology.orchestra.run.smithi105:> sudo mkdir -p /var/lib/ceph/osd/ceph-2 2024-09-21T01:44:22.048 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-09-21T01:44:22.048 INFO:tasks.ceph:role: osd.2 2024-09-21T01:44:22.048 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@smithi105.front.sepia.ceph.com 2024-09-21T01:44:22.048 DEBUG:teuthology.orchestra.run.smithi105:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2024-09-21T01:44:22.120 INFO:teuthology.orchestra.run.smithi105.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=5859072 blks 2024-09-21T01:44:22.120 INFO:teuthology.orchestra.run.smithi105.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-21T01:44:22.120 INFO:teuthology.orchestra.run.smithi105.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-21T01:44:22.120 INFO:teuthology.orchestra.run.smithi105.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-21T01:44:22.120 INFO:teuthology.orchestra.run.smithi105.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-21T01:44:22.121 INFO:teuthology.orchestra.run.smithi105.stdout: = sunit=0 swidth=0 blks 2024-09-21T01:44:22.121 INFO:teuthology.orchestra.run.smithi105.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-21T01:44:22.121 INFO:teuthology.orchestra.run.smithi105.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-21T01:44:22.121 INFO:teuthology.orchestra.run.smithi105.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-21T01:44:22.121 INFO:teuthology.orchestra.run.smithi105.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-21T01:44:22.609 INFO:teuthology.orchestra.run.smithi105.stdout:Discarding blocks...Done. 2024-09-21T01:44:22.611 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@smithi105.front.sepia.ceph.com -o noatime 2024-09-21T01:44:22.611 DEBUG:teuthology.orchestra.run.smithi105:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-2 2024-09-21T01:44:22.765 DEBUG:teuthology.orchestra.run.smithi105:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-2 2024-09-21T01:44:22.794 DEBUG:teuthology.orchestra.run.smithi105:> sudo mkdir -p /var/lib/ceph/osd/ceph-3 2024-09-21T01:44:22.858 INFO:tasks.ceph:roles_to_devs: {'osd.0': '/dev/vg_nvme/lv_1', 'osd.1': '/dev/vg_nvme/lv_2', 'osd.2': '/dev/vg_nvme/lv_3', 'osd.3': '/dev/vg_nvme/lv_4'} 2024-09-21T01:44:22.858 INFO:tasks.ceph:role: osd.3 2024-09-21T01:44:22.859 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_4 on ubuntu@smithi105.front.sepia.ceph.com 2024-09-21T01:44:22.859 DEBUG:teuthology.orchestra.run.smithi105:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_4 2024-09-21T01:44:22.933 INFO:teuthology.orchestra.run.smithi105.stdout:meta-data=/dev/vg_nvme/lv_4 isize=2048 agcount=4, agsize=5859072 blks 2024-09-21T01:44:22.933 INFO:teuthology.orchestra.run.smithi105.stdout: = sectsz=512 attr=2, projid32bit=1 2024-09-21T01:44:22.934 INFO:teuthology.orchestra.run.smithi105.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2024-09-21T01:44:22.934 INFO:teuthology.orchestra.run.smithi105.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2024-09-21T01:44:22.934 INFO:teuthology.orchestra.run.smithi105.stdout:data = bsize=4096 blocks=23436288, imaxpct=25 2024-09-21T01:44:22.934 INFO:teuthology.orchestra.run.smithi105.stdout: = sunit=0 swidth=0 blks 2024-09-21T01:44:22.934 INFO:teuthology.orchestra.run.smithi105.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2024-09-21T01:44:22.934 INFO:teuthology.orchestra.run.smithi105.stdout:log =internal log bsize=4096 blocks=16384, version=2 2024-09-21T01:44:22.934 INFO:teuthology.orchestra.run.smithi105.stdout: = sectsz=512 sunit=0 blks, lazy-count=1 2024-09-21T01:44:22.934 INFO:teuthology.orchestra.run.smithi105.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2024-09-21T01:44:23.441 INFO:teuthology.orchestra.run.smithi105.stdout:Discarding blocks...Done. 2024-09-21T01:44:23.443 INFO:tasks.ceph:mount /dev/vg_nvme/lv_4 on ubuntu@smithi105.front.sepia.ceph.com -o noatime 2024-09-21T01:44:23.444 DEBUG:teuthology.orchestra.run.smithi105:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-3 2024-09-21T01:44:23.593 DEBUG:teuthology.orchestra.run.smithi105:> sudo /sbin/restorecon /var/lib/ceph/osd/ceph-3 2024-09-21T01:44:23.627 DEBUG:teuthology.orchestra.run.smithi105:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-09-21T01:44:23.726 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:23.725+0000 7f823ecfe640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-0/keyring: can't open /var/lib/ceph/osd/ceph-0/keyring: (2) No such file or directory 2024-09-21T01:44:23.727 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:23.726+0000 7f823ecfe640 -1 created new key in keyring /var/lib/ceph/osd/ceph-0/keyring 2024-09-21T01:44:23.727 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:23.726+0000 7f823ecfe640 -1 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-0/block: (2) No such file or directory 2024-09-21T01:44:23.727 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:23.726+0000 7f823ecfe640 -1 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-0/block: (2) No such file or directory 2024-09-21T01:44:23.727 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:23.726+0000 7f823ecfe640 -1 bluestore(/var/lib/ceph/osd/ceph-0/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-0/block: (2) No such file or directory 2024-09-21T01:44:23.727 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:23.726+0000 7f823ecfe640 -1 bluestore(/var/lib/ceph/osd/ceph-0) _read_fsid unparsable uuid 2024-09-21T01:44:26.591 DEBUG:teuthology.orchestra.run.smithi105:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T01:44:26.620 DEBUG:teuthology.orchestra.run.smithi105:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-09-21T01:44:26.710 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:26.709+0000 7f47d87cc640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-1/keyring: can't open /var/lib/ceph/osd/ceph-1/keyring: (2) No such file or directory 2024-09-21T01:44:26.711 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:26.709+0000 7f47d87cc640 -1 created new key in keyring /var/lib/ceph/osd/ceph-1/keyring 2024-09-21T01:44:26.711 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:26.709+0000 7f47d87cc640 -1 bluestore(/var/lib/ceph/osd/ceph-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-1/block: (2) No such file or directory 2024-09-21T01:44:26.711 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:26.709+0000 7f47d87cc640 -1 bluestore(/var/lib/ceph/osd/ceph-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-1/block: (2) No such file or directory 2024-09-21T01:44:26.711 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:26.709+0000 7f47d87cc640 -1 bluestore(/var/lib/ceph/osd/ceph-1/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-1/block: (2) No such file or directory 2024-09-21T01:44:26.711 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:26.709+0000 7f47d87cc640 -1 bluestore(/var/lib/ceph/osd/ceph-1) _read_fsid unparsable uuid 2024-09-21T01:44:29.552 DEBUG:teuthology.orchestra.run.smithi105:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T01:44:29.579 DEBUG:teuthology.orchestra.run.smithi105:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-09-21T01:44:29.667 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:29.665+0000 7f3e370a5640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-2/keyring: can't open /var/lib/ceph/osd/ceph-2/keyring: (2) No such file or directory 2024-09-21T01:44:29.667 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:29.665+0000 7f3e370a5640 -1 created new key in keyring /var/lib/ceph/osd/ceph-2/keyring 2024-09-21T01:44:29.667 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:29.665+0000 7f3e370a5640 -1 bluestore(/var/lib/ceph/osd/ceph-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-2/block: (2) No such file or directory 2024-09-21T01:44:29.667 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:29.665+0000 7f3e370a5640 -1 bluestore(/var/lib/ceph/osd/ceph-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-2/block: (2) No such file or directory 2024-09-21T01:44:29.667 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:29.665+0000 7f3e370a5640 -1 bluestore(/var/lib/ceph/osd/ceph-2/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-2/block: (2) No such file or directory 2024-09-21T01:44:29.667 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:29.666+0000 7f3e370a5640 -1 bluestore(/var/lib/ceph/osd/ceph-2) _read_fsid unparsable uuid 2024-09-21T01:44:32.531 DEBUG:teuthology.orchestra.run.smithi105:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T01:44:32.560 DEBUG:teuthology.orchestra.run.smithi105:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster ceph --mkfs --mkkey -i 3 --monmap /home/ubuntu/cephtest/ceph.monmap 2024-09-21T01:44:32.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:32.649+0000 7f6fdfe1b640 -1 auth: error reading file: /var/lib/ceph/osd/ceph-3/keyring: can't open /var/lib/ceph/osd/ceph-3/keyring: (2) No such file or directory 2024-09-21T01:44:32.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:32.649+0000 7f6fdfe1b640 -1 created new key in keyring /var/lib/ceph/osd/ceph-3/keyring 2024-09-21T01:44:32.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:32.649+0000 7f6fdfe1b640 -1 bluestore(/var/lib/ceph/osd/ceph-3/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-3/block: (2) No such file or directory 2024-09-21T01:44:32.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:32.649+0000 7f6fdfe1b640 -1 bluestore(/var/lib/ceph/osd/ceph-3/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-3/block: (2) No such file or directory 2024-09-21T01:44:32.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:32.649+0000 7f6fdfe1b640 -1 bluestore(/var/lib/ceph/osd/ceph-3/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-3/block: (2) No such file or directory 2024-09-21T01:44:32.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:32.650+0000 7f6fdfe1b640 -1 bluestore(/var/lib/ceph/osd/ceph-3) _read_fsid unparsable uuid 2024-09-21T01:44:35.513 DEBUG:teuthology.orchestra.run.smithi105:> sudo chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T01:44:35.541 INFO:tasks.ceph:Reading keys from all nodes... 2024-09-21T01:44:35.542 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:44:35.542 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/var/lib/ceph/mgr/ceph-x/keyring of=/dev/stdout 2024-09-21T01:44:35.610 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:44:35.610 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/var/lib/ceph/mgr/ceph-y/keyring of=/dev/stdout 2024-09-21T01:44:35.677 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:44:35.677 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/var/lib/ceph/osd/ceph-0/keyring of=/dev/stdout 2024-09-21T01:44:35.743 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:44:35.743 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/var/lib/ceph/osd/ceph-1/keyring of=/dev/stdout 2024-09-21T01:44:35.808 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:44:35.809 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/var/lib/ceph/osd/ceph-2/keyring of=/dev/stdout 2024-09-21T01:44:35.874 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:44:35.874 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/var/lib/ceph/osd/ceph-3/keyring of=/dev/stdout 2024-09-21T01:44:35.939 INFO:tasks.ceph:Adding keys to all mons... 2024-09-21T01:44:35.939 DEBUG:teuthology.orchestra.run.smithi105:> sudo tee -a /etc/ceph/ceph.keyring 2024-09-21T01:44:36.002 INFO:teuthology.orchestra.run.smithi105.stdout:[mgr.x] 2024-09-21T01:44:36.002 INFO:teuthology.orchestra.run.smithi105.stdout: key = AQD0JO5mxkaIGRAAf2+rk8ssnI9JmsbhFtr+Fg== 2024-09-21T01:44:36.002 INFO:teuthology.orchestra.run.smithi105.stdout:[mgr.y] 2024-09-21T01:44:36.002 INFO:teuthology.orchestra.run.smithi105.stdout: key = AQD0JO5mWsqjHBAAnGqmzd0ZUGOTRfjBtOI3EQ== 2024-09-21T01:44:36.002 INFO:teuthology.orchestra.run.smithi105.stdout:[osd.0] 2024-09-21T01:44:36.003 INFO:teuthology.orchestra.run.smithi105.stdout: key = AQD3JO5m+TJcKxAATkhA6clDPIIYl60BBszEIw== 2024-09-21T01:44:36.003 INFO:teuthology.orchestra.run.smithi105.stdout:[osd.1] 2024-09-21T01:44:36.003 INFO:teuthology.orchestra.run.smithi105.stdout: key = AQD6JO5mA1JgKhAAbgBxgFFWRZTekGX/XpRpeA== 2024-09-21T01:44:36.003 INFO:teuthology.orchestra.run.smithi105.stdout:[osd.2] 2024-09-21T01:44:36.003 INFO:teuthology.orchestra.run.smithi105.stdout: key = AQD9JO5mpJ7IJxAAF3czusACg27XYepoUwQrCA== 2024-09-21T01:44:36.003 INFO:teuthology.orchestra.run.smithi105.stdout:[osd.3] 2024-09-21T01:44:36.003 INFO:teuthology.orchestra.run.smithi105.stdout: key = AQAAJe5m9VTFJhAA5hCHWjYza3E9jEGKmw3Rkg== 2024-09-21T01:44:36.004 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-09-21T01:44:36.095 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=mgr.y --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2024-09-21T01:44:36.145 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-21T01:44:36.195 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-21T01:44:36.245 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-21T01:44:36.295 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.3 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2024-09-21T01:44:36.344 INFO:tasks.ceph:Running mkfs on mon nodes... 2024-09-21T01:44:36.344 DEBUG:teuthology.orchestra.run.smithi105:> sudo mkdir -p /var/lib/ceph/mon/ceph-a 2024-09-21T01:44:36.369 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i a --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2024-09-21T01:44:36.464 DEBUG:teuthology.orchestra.run.smithi105:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-a 2024-09-21T01:44:36.488 DEBUG:teuthology.orchestra.run.smithi105:> sudo mkdir -p /var/lib/ceph/mon/ceph-b 2024-09-21T01:44:36.552 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i b --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2024-09-21T01:44:36.646 DEBUG:teuthology.orchestra.run.smithi105:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-b 2024-09-21T01:44:36.670 DEBUG:teuthology.orchestra.run.smithi105:> sudo mkdir -p /var/lib/ceph/mon/ceph-c 2024-09-21T01:44:36.734 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster ceph --mkfs -i c --monmap /home/ubuntu/cephtest/ceph.monmap --keyring /etc/ceph/ceph.keyring 2024-09-21T01:44:36.830 DEBUG:teuthology.orchestra.run.smithi105:> sudo chown -R ceph:ceph /var/lib/ceph/mon/ceph-c 2024-09-21T01:44:36.855 DEBUG:teuthology.orchestra.run.smithi105:> rm -- /home/ubuntu/cephtest/ceph.monmap 2024-09-21T01:44:36.910 INFO:tasks.ceph:Starting mon daemons in cluster ceph... 2024-09-21T01:44:36.910 INFO:tasks.ceph.mon.a:Restarting daemon 2024-09-21T01:44:36.910 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i a 2024-09-21T01:44:36.952 INFO:tasks.ceph.mon.a:Started 2024-09-21T01:44:36.952 INFO:tasks.ceph.mon.b:Restarting daemon 2024-09-21T01:44:36.952 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i b 2024-09-21T01:44:36.954 INFO:tasks.ceph.mon.b:Started 2024-09-21T01:44:36.954 INFO:tasks.ceph.mon.c:Restarting daemon 2024-09-21T01:44:36.955 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i c 2024-09-21T01:44:36.957 INFO:tasks.ceph.mon.c:Started 2024-09-21T01:44:36.957 INFO:tasks.ceph:Starting mgr daemons in cluster ceph... 2024-09-21T01:44:36.957 INFO:tasks.ceph.mgr.x:Restarting daemon 2024-09-21T01:44:36.957 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i x 2024-09-21T01:44:36.959 INFO:tasks.ceph.mgr.x:Started 2024-09-21T01:44:36.959 INFO:tasks.ceph.mgr.y:Restarting daemon 2024-09-21T01:44:36.959 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i y 2024-09-21T01:44:36.961 INFO:tasks.ceph.mgr.y:Started 2024-09-21T01:44:36.961 INFO:tasks.ceph:setting config [client] client_mount_timeout = 600 2024-09-21T01:44:36.961 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set client client_mount_timeout 600 2024-09-21T01:44:36.963 INFO:tasks.ceph:setting config [client] debug_client = 20 2024-09-21T01:44:36.963 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set client debug_client 20 2024-09-21T01:44:37.006 INFO:tasks.ceph:setting config [client] debug_ms = 1 2024-09-21T01:44:37.006 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set client debug_ms 1 2024-09-21T01:44:37.051 INFO:tasks.ceph:setting config [client] rados_mon_op_timeout = 900 2024-09-21T01:44:37.051 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set client rados_mon_op_timeout 900 2024-09-21T01:44:37.094 INFO:tasks.ceph:setting config [client] rados_osd_op_timeout = 900 2024-09-21T01:44:37.094 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set client rados_osd_op_timeout 900 2024-09-21T01:44:37.137 INFO:tasks.ceph:setting config [mds] debug_mds = 20 2024-09-21T01:44:37.137 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set mds debug_mds 20 2024-09-21T01:44:37.180 INFO:tasks.ceph:setting config [mds] debug_mds_balancer = 20 2024-09-21T01:44:37.180 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set mds debug_mds_balancer 20 2024-09-21T01:44:37.223 INFO:tasks.ceph:setting config [mds] debug_ms = 2 2024-09-21T01:44:37.223 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set mds debug_ms 2 2024-09-21T01:44:37.266 INFO:tasks.ceph:setting config [mds] mds_debug_frag = True 2024-09-21T01:44:37.266 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set mds mds_debug_frag True 2024-09-21T01:44:37.309 INFO:tasks.ceph:setting config [mds] mds_debug_scatterstat = True 2024-09-21T01:44:37.309 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set mds mds_debug_scatterstat True 2024-09-21T01:44:37.352 INFO:tasks.ceph:setting config [mds] mds_op_complaint_time = 180 2024-09-21T01:44:37.352 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set mds mds_op_complaint_time 180 2024-09-21T01:44:37.401 INFO:tasks.ceph:setting config [mds] mds_verify_scatter = True 2024-09-21T01:44:37.401 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set mds mds_verify_scatter True 2024-09-21T01:44:37.445 INFO:tasks.ceph:setting config [mds] osd_op_complaint_time = 180 2024-09-21T01:44:37.445 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set mds osd_op_complaint_time 180 2024-09-21T01:44:37.488 INFO:tasks.ceph:setting config [mds] rados_mon_op_timeout = 900 2024-09-21T01:44:37.489 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set mds rados_mon_op_timeout 900 2024-09-21T01:44:37.510 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.510+0000 7f1133fff640 1 -- 172.21.15.105:0/2937782010 <== mon.1 v2:172.21.15.105:3301/0 10 ==== mon_command_ack([{prefix=config set, name=debug_ms}]=0 v4)=0 v4) v1 ==== 109+0+0 (secure 0 0 0) 0x7f113c028020 con 0x7f11340047c0 2024-09-21T01:44:37.516 INFO:tasks.ceph:setting config [mds] rados_osd_op_timeout = 900 2024-09-21T01:44:37.516 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set mds rados_osd_op_timeout 900 2024-09-21T01:44:37.517 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.513+0000 7f1131ffb640 1 -- 172.21.15.105:0/2937782010 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f11340047c0 msgr2=0x7f114c11e750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.517 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.513+0000 7f1131ffb640 1 --2- 172.21.15.105:0/2937782010 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f11340047c0 0x7f114c11e750 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f113c00e600 tx=0x7f113c00eac0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.517 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.513+0000 7f1131ffb640 1 -- 172.21.15.105:0/2937782010 shutdown_connections 2024-09-21T01:44:37.517 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.513+0000 7f1131ffb640 1 --2- 172.21.15.105:0/2937782010 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f114c12be70 0x7f114c1268e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.517 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.513+0000 7f1131ffb640 1 --2- 172.21.15.105:0/2937782010 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f114c1259d0 0x7f114c11ec90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.517 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.513+0000 7f1131ffb640 1 --2- 172.21.15.105:0/2937782010 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f11340047c0 0x7f114c11e750 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.517 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.513+0000 7f1131ffb640 1 -- 172.21.15.105:0/2937782010 >> 172.21.15.105:0/2937782010 conn(0x7f114c099740 msgr2=0x7f114c123480 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:37.517 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.513+0000 7f1131ffb640 1 -- 172.21.15.105:0/2937782010 shutdown_connections 2024-09-21T01:44:37.517 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.513+0000 7f1131ffb640 1 -- 172.21.15.105:0/2937782010 wait complete. 2024-09-21T01:44:37.518 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.517+0000 7f584ffff640 1 -- 172.21.15.105:0/1388508531 <== mon.0 v2:172.21.15.105:3300/0 10 ==== config(4 keys) v1 ==== 110+0+0 (secure 0 0 0) 0x7f5850022880 con 0x7f58581c0f90 2024-09-21T01:44:37.518 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.517+0000 7fe771ffb640 1 -- 172.21.15.105:0/1057063143 <== mon.0 v2:172.21.15.105:3300/0 10 ==== config(4 keys) v1 ==== 110+0+0 (secure 0 0 0) 0x7fe77403cd50 con 0x7fe77c12e7e0 2024-09-21T01:44:37.520 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.520+0000 7f44777fe640 1 -- 172.21.15.105:0/3310372505 <== mon.1 v2:172.21.15.105:3301/0 10 ==== config(4 keys) v1 ==== 110+0+0 (secure 0 0 0) 0x7f4488033d40 con 0x7f449012c680 2024-09-21T01:44:37.520 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.520+0000 7fb184d33640 1 -- 172.21.15.105:0/1965238775 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb18009ad20 msgr2=0x7fb18008c020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.520 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.520+0000 7fb184d33640 1 --2- 172.21.15.105:0/1965238775 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb18009ad20 0x7fb18008c020 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7fb178005fd0 tx=0x7fb17802ab70 comp rx=0 tx=0).stop 2024-09-21T01:44:37.521 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.520+0000 7fb184d33640 1 -- 172.21.15.105:0/1965238775 shutdown_connections 2024-09-21T01:44:37.521 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.520+0000 7fb184d33640 1 --2- 172.21.15.105:0/1965238775 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb18008caa0 0x7fb180095f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.521 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.520+0000 7fb184d33640 1 --2- 172.21.15.105:0/1965238775 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb18006d030 0x7fb18008c560 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.521 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.520+0000 7fb184d33640 1 --2- 172.21.15.105:0/1965238775 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb18009ad20 0x7fb18008c020 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.521 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.520+0000 7fb184d33640 1 -- 172.21.15.105:0/1965238775 >> 172.21.15.105:0/1965238775 conn(0x7fb180098b10 msgr2=0x7fb18009a610 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:37.522 INFO:tasks.ceph:setting config [mgr] client_mount_timeout = 30 2024-09-21T01:44:37.522 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set mgr client_mount_timeout 30 2024-09-21T01:44:37.523 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.522+0000 7f100dffb640 1 -- 172.21.15.105:0/165582344 <== mon.1 v2:172.21.15.105:3301/0 10 ==== config(4 keys) v1 ==== 110+0+0 (secure 0 0 0) 0x7f1014010820 con 0x7f1004005220 2024-09-21T01:44:37.525 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.524+0000 7f584ffff640 1 -- 172.21.15.105:0/1388508531 <== mon.0 v2:172.21.15.105:3300/0 11 ==== mon_command_ack([{prefix=config set, name=rados_osd_op_timeout}]=0 v5)=0 v5) v1 ==== 123+0+0 (secure 0 0 0) 0x7f5850022b70 con 0x7f58581c0f90 2024-09-21T01:44:37.525 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.521+0000 7fb184d33640 1 -- 172.21.15.105:0/1965238775 shutdown_connections 2024-09-21T01:44:37.525 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.521+0000 7fb184d33640 1 -- 172.21.15.105:0/1965238775 wait complete. 2024-09-21T01:44:37.525 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.523+0000 7fb184d33640 1 Processor -- start 2024-09-21T01:44:37.526 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.523+0000 7fb184d33640 1 -- start start 2024-09-21T01:44:37.526 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.523+0000 7fb184d33640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb18006d030 0x7fb18007b190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.526 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.523+0000 7fb184d33640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb18008caa0 0x7fb18007b6d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.526 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.523+0000 7fb184d33640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb18007bc10 0x7fb1801d44a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.526 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.523+0000 7fb184d33640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb18007d1e0 con 0x7fb18007bc10 2024-09-21T01:44:37.526 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.523+0000 7fb184d33640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb18007dde0 con 0x7fb18008caa0 2024-09-21T01:44:37.526 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.523+0000 7fb184d33640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fb18007d620 con 0x7fb18006d030 2024-09-21T01:44:37.526 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.524+0000 7fb17dd9b640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb18008caa0 0x7fb18007b6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:37.526 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.524+0000 7fb17dd9b640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb18008caa0 0x7fb18007b6d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:50162/0 (socket says 172.21.15.105:50162) 2024-09-21T01:44:37.526 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.524+0000 7fb17dd9b640 1 -- 172.21.15.105:0/1041121881 learned_addr learned my addr 172.21.15.105:0/1041121881 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:37.526 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.524+0000 7fb17dd9b640 1 -- 172.21.15.105:0/1041121881 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb18006d030 msgr2=0x7fb18007b190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.526 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.524+0000 7fb17dd9b640 1 --2- 172.21.15.105:0/1041121881 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb18006d030 0x7fb18007b190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.526 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.524+0000 7fb17dd9b640 1 -- 172.21.15.105:0/1041121881 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb18007bc10 msgr2=0x7fb1801d44a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.527 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.524+0000 7fb17dd9b640 1 --2- 172.21.15.105:0/1041121881 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb18007bc10 0x7fb1801d44a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.527 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.524+0000 7fb17dd9b640 1 -- 172.21.15.105:0/1041121881 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb178009c90 con 0x7fb18008caa0 2024-09-21T01:44:37.527 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.525+0000 7fb17dd9b640 1 --2- 172.21.15.105:0/1041121881 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb18008caa0 0x7fb18007b6d0 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7fb17000d780 tx=0x7fb17000dc40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:37.527 INFO:tasks.ceph:setting config [mgr] debug_client = 20 2024-09-21T01:44:37.528 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set mgr debug_client 20 2024-09-21T01:44:37.528 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.527+0000 7f585d8f7640 1 -- 172.21.15.105:0/1388508531 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f58581c0f90 msgr2=0x7f58581c13e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.528 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.527+0000 7f585d8f7640 1 --2- 172.21.15.105:0/1388508531 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f58581c0f90 0x7f58581c13e0 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f5858066710 tx=0x7f585000fe30 comp rx=0 tx=0).stop 2024-09-21T01:44:37.528 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.527+0000 7f585d8f7640 1 -- 172.21.15.105:0/1388508531 shutdown_connections 2024-09-21T01:44:37.528 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.527+0000 7f585d8f7640 1 --2- 172.21.15.105:0/1388508531 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f58581c0f90 0x7f58581c13e0 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.528 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.527+0000 7f585d8f7640 1 --2- 172.21.15.105:0/1388508531 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f585806e820 0x7f585806ec60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.528 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.527+0000 7f585d8f7640 1 --2- 172.21.15.105:0/1388508531 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f5844003a40 0x7f585806e2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.528 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.527+0000 7f585d8f7640 1 -- 172.21.15.105:0/1388508531 >> 172.21.15.105:0/1388508531 conn(0x7f585809af10 msgr2=0x7f585812e3a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:37.531 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.527+0000 7fa13cff9640 1 -- 172.21.15.105:0/1500671650 <== mon.2 v2:172.21.15.105:3302/0 9 ==== config(4 keys) v1 ==== 110+0+0 (secure 0 0 0) 0x7fa13804b710 con 0x7fa14012c010 2024-09-21T01:44:37.531 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.528+0000 7fb16effd640 1 -- 172.21.15.105:0/1041121881 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb170009c30 con 0x7fb18008caa0 2024-09-21T01:44:37.531 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.528+0000 7fb16effd640 1 -- 172.21.15.105:0/1041121881 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(4 keys) v1 ==== 110+0+0 (secure 0 0 0) 0x7fb170009e60 con 0x7fb18008caa0 2024-09-21T01:44:37.531 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.528+0000 7fb16effd640 1 -- 172.21.15.105:0/1041121881 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb1700106c0 con 0x7fb18008caa0 2024-09-21T01:44:37.531 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.528+0000 7fb184d33640 1 -- 172.21.15.105:0/1041121881 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb18007c9b0 con 0x7fb18008caa0 2024-09-21T01:44:37.531 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.528+0000 7fb184d33640 1 -- 172.21.15.105:0/1041121881 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb18007c040 con 0x7fb18008caa0 2024-09-21T01:44:37.532 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.531+0000 7fb16cff9640 1 -- 172.21.15.105:0/1041121881 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb164005040 con 0x7fb18008caa0 2024-09-21T01:44:37.532 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.532+0000 7f585d8f7640 1 -- 172.21.15.105:0/1388508531 shutdown_connections 2024-09-21T01:44:37.532 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.532+0000 7f585d8f7640 1 -- 172.21.15.105:0/1388508531 wait complete. 2024-09-21T01:44:37.534 INFO:tasks.ceph:setting config [mgr] debug_mgr = 20 2024-09-21T01:44:37.534 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set mgr debug_mgr 20 2024-09-21T01:44:37.541 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.538+0000 7f100dffb640 1 -- 172.21.15.105:0/165582344 <== mon.1 v2:172.21.15.105:3301/0 11 ==== mon_command_ack([{prefix=config set, name=debug_mds}]=0 v6)=0 v6) v1 ==== 108+0+0 (secure 0 0 0) 0x7f1014029070 con 0x7f1004005220 2024-09-21T01:44:37.542 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.538+0000 7fb16effd640 1 -- 172.21.15.105:0/1041121881 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7fb170010b20 con 0x7fb18008caa0 2024-09-21T01:44:37.542 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.538+0000 7fb16effd640 1 -- 172.21.15.105:0/1041121881 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7fb17000f3c0 con 0x7fb18008caa0 2024-09-21T01:44:37.542 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.538+0000 7fb16effd640 1 -- 172.21.15.105:0/1041121881 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7fb17003be30 con 0x7fb18008caa0 2024-09-21T01:44:37.544 INFO:tasks.ceph:setting config [mgr] debug_ms = 1 2024-09-21T01:44:37.545 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set mgr debug_ms 1 2024-09-21T01:44:37.545 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.541+0000 7f101f353640 1 -- 172.21.15.105:0/165582344 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f1004005220 msgr2=0x7f1018123320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.545 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.541+0000 7f101f353640 1 --2- 172.21.15.105:0/165582344 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f1004005220 0x7f1018123320 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f101400d720 tx=0x7f101400dbe0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.545 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.542+0000 7f101f353640 1 -- 172.21.15.105:0/165582344 shutdown_connections 2024-09-21T01:44:37.545 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.542+0000 7f101f353640 1 --2- 172.21.15.105:0/165582344 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1018123da0 0x7f10181c2b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.545 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.542+0000 7f101f353640 1 --2- 172.21.15.105:0/165582344 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f101808f430 0x7f1018123860 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.546 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.542+0000 7f101f353640 1 --2- 172.21.15.105:0/165582344 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f1004005220 0x7f1018123320 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.546 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.542+0000 7f101f353640 1 -- 172.21.15.105:0/165582344 >> 172.21.15.105:0/165582344 conn(0x7f101809a830 msgr2=0x7f10181277e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:37.546 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.542+0000 7f101f353640 1 -- 172.21.15.105:0/165582344 shutdown_connections 2024-09-21T01:44:37.546 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.542+0000 7f101f353640 1 -- 172.21.15.105:0/165582344 wait complete. 2024-09-21T01:44:37.559 INFO:tasks.ceph:setting config [mgr] mon_warn_on_pool_no_app = False 2024-09-21T01:44:37.559 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set mgr mon_warn_on_pool_no_app False 2024-09-21T01:44:37.583 INFO:tasks.ceph:setting config [mon] mon_op_complaint_time = 120 2024-09-21T01:44:37.583 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set mon mon_op_complaint_time 120 2024-09-21T01:44:37.606 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:37.606+0000 7f43c91dd040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T01:44:37.609 INFO:tasks.ceph:setting config [osd] osd_op_complaint_time = 180 2024-09-21T01:44:37.609 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set osd osd_op_complaint_time 180 2024-09-21T01:44:37.616 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:37.616+0000 7fc75c220040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T01:44:37.619 DEBUG:tasks.ceph:set 22 configs 2024-09-21T01:44:37.619 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'client_mount_timeout', '600'], check_status=True, hostname='smithi105') 2024-09-21T01:44:37.619 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'debug_client', '20'], check_status=True, hostname='smithi105') 2024-09-21T01:44:37.619 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'debug_ms', '1'], check_status=True, hostname='smithi105') 2024-09-21T01:44:37.619 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'rados_mon_op_timeout', '900'], check_status=True, hostname='smithi105') 2024-09-21T01:44:37.626 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.623+0000 7f0b2a701640 1 -- 172.21.15.105:0/2178899453 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0b241289d0 msgr2=0x7f0b24128db0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.626 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.623+0000 7f0b2a701640 1 --2- 172.21.15.105:0/2178899453 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0b241289d0 0x7f0b24128db0 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f0b140098b0 tx=0x7f0b1402a340 comp rx=0 tx=0).stop 2024-09-21T01:44:37.626 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.624+0000 7f0b2a701640 1 -- 172.21.15.105:0/2178899453 shutdown_connections 2024-09-21T01:44:37.626 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.624+0000 7f0b2a701640 1 --2- 172.21.15.105:0/2178899453 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0b2412d9a0 0x7f0b2412fd90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.627 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.624+0000 7f0b2a701640 1 --2- 172.21.15.105:0/2178899453 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0b241289d0 0x7f0b24128db0 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.627 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.624+0000 7f0b2a701640 1 -- 172.21.15.105:0/2178899453 >> 172.21.15.105:0/2178899453 conn(0x7f0b2409aa30 msgr2=0x7f0b2409ae40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:37.627 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.624+0000 7f0b2a701640 1 -- 172.21.15.105:0/2178899453 shutdown_connections 2024-09-21T01:44:37.627 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.624+0000 7f0b2a701640 1 -- 172.21.15.105:0/2178899453 wait complete. 2024-09-21T01:44:37.627 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.625+0000 7f0b2a701640 1 Processor -- start 2024-09-21T01:44:37.627 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.625+0000 7f0b2a701640 1 -- start start 2024-09-21T01:44:37.629 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.626+0000 7f0b2a701640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0b241289d0 0x7f0b241315e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.629 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.626+0000 7f0b2a701640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0b241292f0 0x7f0b24131b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.629 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.626+0000 7f0b2a701640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0b2412d9a0 0x7f0b24132060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.629 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.626+0000 7f0b2a701640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0b2412ad50 con 0x7f0b2412d9a0 2024-09-21T01:44:37.629 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.626+0000 7f0b2a701640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0b2412a570 con 0x7f0b241289d0 2024-09-21T01:44:37.629 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.626+0000 7f0b2a701640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f0b2412d550 con 0x7f0b241292f0 2024-09-21T01:44:37.631 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.629+0000 7f0b237fe640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0b241292f0 0x7f0b24131b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:37.631 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.631+0000 7f0b237fe640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0b241292f0 0x7f0b24131b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:45464/0 (socket says 172.21.15.105:45464) 2024-09-21T01:44:37.631 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.631+0000 7f0b237fe640 1 -- 172.21.15.105:0/672714383 learned_addr learned my addr 172.21.15.105:0/672714383 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:37.638 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.632+0000 7f0b237fe640 1 -- 172.21.15.105:0/672714383 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0b241289d0 msgr2=0x7f0b241315e0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:44:37.638 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.632+0000 7f0b237fe640 1 --2- 172.21.15.105:0/672714383 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0b241289d0 0x7f0b241315e0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.638 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.632+0000 7f0b237fe640 1 -- 172.21.15.105:0/672714383 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0b2412d9a0 msgr2=0x7f0b24132060 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:44:37.638 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.632+0000 7f0b237fe640 1 --2- 172.21.15.105:0/672714383 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0b2412d9a0 0x7f0b24132060 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.638 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.632+0000 7f0b237fe640 1 -- 172.21.15.105:0/672714383 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0b14009490 con 0x7f0b241292f0 2024-09-21T01:44:37.639 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.638+0000 7f0b237fe640 1 --2- 172.21.15.105:0/672714383 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0b241292f0 0x7f0b24131b20 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f0b1c00cc60 tx=0x7f0b1c00a430 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:37.645 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.645+0000 7f0b20ff9640 1 -- 172.21.15.105:0/672714383 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b1c00ace0 con 0x7f0b241292f0 2024-09-21T01:44:37.645 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.645+0000 7f0b20ff9640 1 -- 172.21.15.105:0/672714383 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(4 keys) v1 ==== 110+0+0 (secure 0 0 0) 0x7f0b1c00ce70 con 0x7f0b241292f0 2024-09-21T01:44:37.645 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.645+0000 7f0b20ff9640 1 -- 172.21.15.105:0/672714383 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0b1c003bd0 con 0x7f0b241292f0 2024-09-21T01:44:37.645 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.645+0000 7f0b2a701640 1 -- 172.21.15.105:0/672714383 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0b2412afa0 con 0x7f0b241292f0 2024-09-21T01:44:37.646 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.645+0000 7f0b2a701640 1 -- 172.21.15.105:0/672714383 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0b24128110 con 0x7f0b241292f0 2024-09-21T01:44:37.648 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.648+0000 7f0b20ff9640 1 -- 172.21.15.105:0/672714383 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f0b1c005370 con 0x7f0b241292f0 2024-09-21T01:44:37.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.650+0000 7f0b20ff9640 1 -- 172.21.15.105:0/672714383 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f0b1c01cb80 con 0x7f0b241292f0 2024-09-21T01:44:37.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.650+0000 7f0b0a7fc640 1 -- 172.21.15.105:0/672714383 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0b24128db0 con 0x7f0b241292f0 2024-09-21T01:44:37.654 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.654+0000 7f0b20ff9640 1 -- 172.21.15.105:0/672714383 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f0b1c017070 con 0x7f0b241292f0 2024-09-21T01:44:37.665 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.664+0000 7f18fdd2f640 1 -- 172.21.15.105:0/3220331143 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f18f8133570 msgr2=0x7f18f8135960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.665 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.664+0000 7f18fdd2f640 1 --2- 172.21.15.105:0/3220331143 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f18f8133570 0x7f18f8135960 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f18f0005fd0 tx=0x7f18f002a750 comp rx=0 tx=0).stop 2024-09-21T01:44:37.665 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.665+0000 7f18fd52e640 1 --2- 172.21.15.105:0/3220331143 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f18f8135ea0 0x7f18f8138290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:37.665 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.665+0000 7f18fdd2f640 1 -- 172.21.15.105:0/3220331143 shutdown_connections 2024-09-21T01:44:37.665 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.665+0000 7f18fdd2f640 1 --2- 172.21.15.105:0/3220331143 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f18f8135ea0 0x7f18f8138290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.666 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.665+0000 7f18fdd2f640 1 --2- 172.21.15.105:0/3220331143 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f18f8133570 0x7f18f8135960 secure :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f18f0005fd0 tx=0x7f18f002a750 comp rx=0 tx=0).stop 2024-09-21T01:44:37.666 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.665+0000 7f18fdd2f640 1 -- 172.21.15.105:0/3220331143 >> 172.21.15.105:0/3220331143 conn(0x7f18f806bc90 msgr2=0x7f18f806c0a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:37.673 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.673+0000 7f18fdd2f640 1 -- 172.21.15.105:0/3220331143 shutdown_connections 2024-09-21T01:44:37.679 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.674+0000 7f18fdd2f640 1 -- 172.21.15.105:0/3220331143 wait complete. 2024-09-21T01:44:37.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.675+0000 7f18fdd2f640 1 Processor -- start 2024-09-21T01:44:37.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.675+0000 7f18fdd2f640 1 -- start start 2024-09-21T01:44:37.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.676+0000 7f18fdd2f640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f18f812d100 0x7f18f80a7e30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.676+0000 7f18fdd2f640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f18f8135ea0 0x7f18f806e130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.676+0000 7f18fdd2f640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f18f806e670 0x7f18f806eb00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.676+0000 7f18fdd2f640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f18f812a860 con 0x7f18f812d100 2024-09-21T01:44:37.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.676+0000 7f18fdd2f640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f18f806f8c0 con 0x7f18f8135ea0 2024-09-21T01:44:37.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.676+0000 7f18fdd2f640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f18f812aa90 con 0x7f18f806e670 2024-09-21T01:44:37.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.676+0000 7f18fd52e640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f18f806e670 0x7f18f806eb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:37.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.676+0000 7f18fd52e640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f18f806e670 0x7f18f806eb00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:45466/0 (socket says 172.21.15.105:45466) 2024-09-21T01:44:37.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.676+0000 7f18fd52e640 1 -- 172.21.15.105:0/1796095297 learned_addr learned my addr 172.21.15.105:0/1796095297 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:37.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.677+0000 7f18fd52e640 1 -- 172.21.15.105:0/1796095297 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f18f8135ea0 msgr2=0x7f18f806e130 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:44:37.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.677+0000 7f18fd52e640 1 --2- 172.21.15.105:0/1796095297 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f18f8135ea0 0x7f18f806e130 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.681 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.677+0000 7f18fd52e640 1 -- 172.21.15.105:0/1796095297 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f18f812d100 msgr2=0x7f18f80a7e30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.681 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.677+0000 7f18fd52e640 1 --2- 172.21.15.105:0/1796095297 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f18f812d100 0x7f18f80a7e30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.681 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.677+0000 7f18fd52e640 1 -- 172.21.15.105:0/1796095297 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f18ec009560 con 0x7f18f806e670 2024-09-21T01:44:37.681 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.677+0000 7f18fd52e640 1 --2- 172.21.15.105:0/1796095297 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f18f806e670 0x7f18f806eb00 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f18ec00ecb0 tx=0x7f18ec00c6a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:37.681 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.680+0000 7f18f57fa640 1 -- 172.21.15.105:0/1796095297 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f18ec00eda0 con 0x7f18f806e670 2024-09-21T01:44:37.683 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.680+0000 7f18fdd2f640 1 -- 172.21.15.105:0/1796095297 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f18f0009840 con 0x7f18f806e670 2024-09-21T01:44:37.683 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.680+0000 7f18fdd2f640 1 -- 172.21.15.105:0/1796095297 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f18f806f2d0 con 0x7f18f806e670 2024-09-21T01:44:37.683 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.681+0000 7f18f57fa640 1 -- 172.21.15.105:0/1796095297 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(4 keys) v1 ==== 110+0+0 (secure 0 0 0) 0x7f18ec004030 con 0x7f18f806e670 2024-09-21T01:44:37.683 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.681+0000 7f18f57fa640 1 -- 172.21.15.105:0/1796095297 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f18ec004400 con 0x7f18f806e670 2024-09-21T01:44:37.684 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.683+0000 7f18f57fa640 1 -- 172.21.15.105:0/1796095297 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f18ec0035f0 con 0x7f18f806e670 2024-09-21T01:44:37.684 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.683+0000 7f18f57fa640 1 -- 172.21.15.105:0/1796095297 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f18ec014070 con 0x7f18f806e670 2024-09-21T01:44:37.702 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.699+0000 7f18fdd2f640 1 -- 172.21.15.105:0/1796095297 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f18d8005040 con 0x7f18f806e670 2024-09-21T01:44:37.705 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.705+0000 7f18f57fa640 1 -- 172.21.15.105:0/1796095297 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f18ec021af0 con 0x7f18f806e670 2024-09-21T01:44:37.711 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.711+0000 7f44757fa640 1 -- 172.21.15.105:0/3310372505 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command([{prefix=config set, name=rados_mon_op_timeout}] v 0) v1 -- 0x7f44680050c0 con 0x7f449012c680 2024-09-21T01:44:37.719 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.719+0000 7fe783711640 1 -- 172.21.15.105:0/1057063143 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command([{prefix=config set, name=debug_mds_balancer}] v 0) v1 -- 0x7fe7540050a0 con 0x7fe77c12e7e0 2024-09-21T01:44:37.720 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.719+0000 7fe771ffb640 1 -- 172.21.15.105:0/1057063143 <== mon.0 v2:172.21.15.105:3300/0 11 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fe774045a00 con 0x7fe77c12e7e0 2024-09-21T01:44:37.722 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.722+0000 7fb16effd640 1 -- 172.21.15.105:0/1041121881 <== mon.1 v2:172.21.15.105:3301/0 7 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fb17000de90 con 0x7fb18008caa0 2024-09-21T01:44:37.722 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.722+0000 7f44777fe640 1 -- 172.21.15.105:0/3310372505 <== mon.1 v2:172.21.15.105:3301/0 11 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f4488050430 con 0x7f449012c680 2024-09-21T01:44:37.722 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.722+0000 7f44777fe640 1 -- 172.21.15.105:0/3310372505 <== mon.1 v2:172.21.15.105:3301/0 12 ==== mon_command_ack([{prefix=config set, name=rados_mon_op_timeout}]=0 v7)=0 v7) v1 ==== 123+0+0 (secure 0 0 0) 0x7f44880550c0 con 0x7f449012c680 2024-09-21T01:44:37.725 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.723+0000 7fa13cff9640 1 -- 172.21.15.105:0/1500671650 <== mon.2 v2:172.21.15.105:3302/0 10 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fa13804b980 con 0x7fa14012c010 2024-09-21T01:44:37.726 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.724+0000 7f0b20ff9640 1 -- 172.21.15.105:0/672714383 <== mon.2 v2:172.21.15.105:3302/0 7 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f0b1c005800 con 0x7f0b241292f0 2024-09-21T01:44:37.726 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.725+0000 7f18f57fa640 1 -- 172.21.15.105:0/1796095297 <== mon.2 v2:172.21.15.105:3302/0 7 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f18ec021cd0 con 0x7f18f806e670 2024-09-21T01:44:37.727 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.726+0000 7f4495701640 1 -- 172.21.15.105:0/3310372505 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f449012c680 msgr2=0x7f4490116ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.727 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.726+0000 7f4495701640 1 --2- 172.21.15.105:0/3310372505 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f449012c680 0x7f4490116ad0 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f4488032000 tx=0x7f4488004ab0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.727 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.726+0000 7f4495701640 1 -- 172.21.15.105:0/3310372505 shutdown_connections 2024-09-21T01:44:37.727 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.726+0000 7f4495701640 1 --2- 172.21.15.105:0/3310372505 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f449012c680 0x7f4490116ad0 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.727 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.726+0000 7f4495701640 1 --2- 172.21.15.105:0/3310372505 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f449006ee50 0x7f4490116590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.727 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.726+0000 7f4495701640 1 --2- 172.21.15.105:0/3310372505 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f449006e530 0x7f4490116050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.727 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.726+0000 7f4495701640 1 -- 172.21.15.105:0/3310372505 >> 172.21.15.105:0/3310372505 conn(0x7f44900980a0 msgr2=0x7f449009aa30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:37.727 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.726+0000 7f4495701640 1 -- 172.21.15.105:0/3310372505 shutdown_connections 2024-09-21T01:44:37.727 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.726+0000 7f4495701640 1 -- 172.21.15.105:0/3310372505 wait complete. 2024-09-21T01:44:37.730 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.730+0000 7fe771ffb640 1 -- 172.21.15.105:0/1057063143 <== mon.0 v2:172.21.15.105:3300/0 12 ==== mon_command_ack([{prefix=config set, name=debug_mds_balancer}]=0 v8)=0 v8) v1 ==== 117+0+0 (secure 0 0 0) 0x7fe774040030 con 0x7fe77c12e7e0 2024-09-21T01:44:37.734 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.734+0000 7fe7677fe640 1 -- 172.21.15.105:0/1057063143 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe77c12e7e0 msgr2=0x7fe77c1c51e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.735 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.734+0000 7fe7677fe640 1 --2- 172.21.15.105:0/1057063143 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe77c12e7e0 0x7fe77c1c51e0 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7fe77402ae00 tx=0x7fe774004a40 comp rx=0 tx=0).stop 2024-09-21T01:44:37.735 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.734+0000 7fe7677fe640 1 -- 172.21.15.105:0/1057063143 shutdown_connections 2024-09-21T01:44:37.735 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.734+0000 7fe7677fe640 1 --2- 172.21.15.105:0/1057063143 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe77c12e7e0 0x7fe77c1c51e0 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.735 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.734+0000 7fe7677fe640 1 --2- 172.21.15.105:0/1057063143 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fe77c12beb0 0x7fe77c130110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.735 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.734+0000 7fe7677fe640 1 --2- 172.21.15.105:0/1057063143 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fe77c125b70 0x7fe77c12fbd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.735 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.734+0000 7fe7677fe640 1 -- 172.21.15.105:0/1057063143 >> 172.21.15.105:0/1057063143 conn(0x7fe77c0980a0 msgr2=0x7fe77c06d520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:37.735 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.734+0000 7fe7677fe640 1 -- 172.21.15.105:0/1057063143 shutdown_connections 2024-09-21T01:44:37.735 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.735+0000 7fe7677fe640 1 -- 172.21.15.105:0/1057063143 wait complete. 2024-09-21T01:44:37.749 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:37.749+0000 7f43c91dd040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T01:44:37.767 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:37.767+0000 7fc75c220040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T01:44:37.777 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'client', 'rados_osd_op_timeout', '900'], check_status=True, hostname='smithi105') 2024-09-21T01:44:37.777 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'debug_mds', '20'], check_status=True, hostname='smithi105') 2024-09-21T01:44:37.777 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'debug_mds_balancer', '20'], check_status=True, hostname='smithi105') 2024-09-21T01:44:37.778 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'debug_ms', '2'], check_status=True, hostname='smithi105') 2024-09-21T01:44:37.798 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.798+0000 7f026a067640 1 -- 172.21.15.105:0/3371178000 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f025c0b0b80 msgr2=0x7f025c0b2f70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.798 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.798+0000 7f026a067640 1 --2- 172.21.15.105:0/3371178000 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f025c0b0b80 0x7f025c0b2f70 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f0264067ac0 tx=0x7f02640a3440 comp rx=0 tx=0).stop 2024-09-21T01:44:37.799 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.798+0000 7f026a067640 1 -- 172.21.15.105:0/3371178000 shutdown_connections 2024-09-21T01:44:37.799 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.798+0000 7f026a067640 1 --2- 172.21.15.105:0/3371178000 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f025c0b0b80 0x7f025c0b2f70 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.799 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.798+0000 7f026a067640 1 --2- 172.21.15.105:0/3371178000 >> v2:172.21.15.105:3300/0 conn(0x7f025c01a660 0x7f025c0b0640 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.799 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.798+0000 7f026a067640 1 -- 172.21.15.105:0/3371178000 >> 172.21.15.105:0/3371178000 conn(0x7f025c005af0 msgr2=0x7f025c005f00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:37.799 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.799+0000 7f026a067640 1 -- 172.21.15.105:0/3371178000 shutdown_connections 2024-09-21T01:44:37.800 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.799+0000 7f026a067640 1 -- 172.21.15.105:0/3371178000 wait complete. 2024-09-21T01:44:37.800 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.800+0000 7f026a067640 1 Processor -- start 2024-09-21T01:44:37.802 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.800+0000 7f026a067640 1 -- start start 2024-09-21T01:44:37.802 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.801+0000 7f026a067640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f025c0b34b0 0x7f025c018a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.802 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.801+0000 7f026a067640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f025c018fb0 0x7f025c0193f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.803 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.801+0000 7f026a067640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f025c151900 0x7f025c153cf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.803 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.801+0000 7f026a067640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f025c0a7dd0 con 0x7f025c0b34b0 2024-09-21T01:44:37.803 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.801+0000 7f026a067640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f025c0a7430 con 0x7f025c018fb0 2024-09-21T01:44:37.803 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.801+0000 7f026a067640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f025c01e710 con 0x7f025c151900 2024-09-21T01:44:37.803 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.801+0000 7f0269866640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f025c151900 0x7f025c153cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:37.803 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.802+0000 7f0268864640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f025c018fb0 0x7f025c0193f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:37.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.802+0000 7f0269065640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f025c0b34b0 0x7f025c018a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:37.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.802+0000 7f0268864640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f025c018fb0 0x7f025c0193f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:50186/0 (socket says 172.21.15.105:50186) 2024-09-21T01:44:37.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.802+0000 7f0268864640 1 -- 172.21.15.105:0/2199543376 learned_addr learned my addr 172.21.15.105:0/2199543376 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:37.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.804+0000 7f0268864640 1 -- 172.21.15.105:0/2199543376 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f025c151900 msgr2=0x7f025c153cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.805 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.804+0000 7f0268864640 1 --2- 172.21.15.105:0/2199543376 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f025c151900 0x7f025c153cf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.805 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.804+0000 7f0268864640 1 -- 172.21.15.105:0/2199543376 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f025c0b34b0 msgr2=0x7f025c018a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.805 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.804+0000 7f0268864640 1 --2- 172.21.15.105:0/2199543376 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f025c0b34b0 0x7f025c018a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.805 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.804+0000 7f0268864640 1 -- 172.21.15.105:0/2199543376 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0264073160 con 0x7f025c018fb0 2024-09-21T01:44:37.805 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.804+0000 7f0269065640 1 --2- 172.21.15.105:0/2199543376 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f025c0b34b0 0x7f025c018a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:37.805 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.804+0000 7f0269866640 1 --2- 172.21.15.105:0/2199543376 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f025c151900 0x7f025c153cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:37.806 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.805+0000 7f0268864640 1 --2- 172.21.15.105:0/2199543376 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f025c018fb0 0x7f025c0193f0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f0264069bd0 tx=0x7f02640a9940 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:37.806 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.805+0000 7f0259ffb640 1 -- 172.21.15.105:0/2199543376 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f02640a9db0 con 0x7f025c018fb0 2024-09-21T01:44:37.806 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.806+0000 7f026a067640 1 -- 172.21.15.105:0/2199543376 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f025c0a7af0 con 0x7f025c018fb0 2024-09-21T01:44:37.807 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.806+0000 7f026a067640 1 -- 172.21.15.105:0/2199543376 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f025c0ad4d0 con 0x7f025c018fb0 2024-09-21T01:44:37.807 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.807+0000 7f0259ffb640 1 -- 172.21.15.105:0/2199543376 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f02640b3430 con 0x7f025c018fb0 2024-09-21T01:44:37.807 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.807+0000 7f0259ffb640 1 -- 172.21.15.105:0/2199543376 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f02640b37a0 con 0x7f025c018fb0 2024-09-21T01:44:37.807 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.807+0000 7f0259ffb640 1 -- 172.21.15.105:0/2199543376 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f02640b3980 con 0x7f025c018fb0 2024-09-21T01:44:37.807 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.807+0000 7f0259ffb640 1 -- 172.21.15.105:0/2199543376 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f02640bdbb0 con 0x7f025c018fb0 2024-09-21T01:44:37.809 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.809+0000 7f026a067640 1 -- 172.21.15.105:0/2199543376 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0244005250 con 0x7f025c018fb0 2024-09-21T01:44:37.813 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.813+0000 7f0259ffb640 1 -- 172.21.15.105:0/2199543376 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f02640b3c30 con 0x7f025c018fb0 2024-09-21T01:44:37.837 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.837+0000 7f92560f8640 1 -- 172.21.15.105:0/3719572508 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f925008c200 msgr2=0x7f92501290a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.837 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.837+0000 7f92560f8640 1 --2- 172.21.15.105:0/3719572508 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f925008c200 0x7f92501290a0 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f9240009830 tx=0x7f924002a3d0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.842 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.842+0000 7f92560f8640 1 -- 172.21.15.105:0/3719572508 shutdown_connections 2024-09-21T01:44:37.842 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.842+0000 7f92560f8640 1 --2- 172.21.15.105:0/3719572508 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f925008c200 0x7f92501290a0 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.842 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.842+0000 7f92560f8640 1 --2- 172.21.15.105:0/3719572508 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9250093080 0x7f9250128b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.842 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.842+0000 7f92560f8640 1 -- 172.21.15.105:0/3719572508 >> 172.21.15.105:0/3719572508 conn(0x7f925009a7f0 msgr2=0x7f925009ac00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:37.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.847+0000 7f92560f8640 1 -- 172.21.15.105:0/3719572508 shutdown_connections 2024-09-21T01:44:37.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.847+0000 7f92560f8640 1 -- 172.21.15.105:0/3719572508 wait complete. 2024-09-21T01:44:37.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.848+0000 7f92560f8640 1 Processor -- start 2024-09-21T01:44:37.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.848+0000 7f92560f8640 1 -- start start 2024-09-21T01:44:37.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.849+0000 7f92560f8640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f925008c200 0x7f92501384b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.849+0000 7f92560f8640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f9250093080 0x7f9250125730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.849+0000 7f92560f8640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f92501295e0 0x7f9250125c70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.849+0000 7f92560f8640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9250092310 con 0x7f925008c200 2024-09-21T01:44:37.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.849+0000 7f92560f8640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9250126740 con 0x7f92501295e0 2024-09-21T01:44:37.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.849+0000 7f92560f8640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f925006cac0 con 0x7f9250093080 2024-09-21T01:44:37.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.849+0000 7f92550f6640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f925008c200 0x7f92501384b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:37.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.849+0000 7f92550f6640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f925008c200 0x7f92501384b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:55376/0 (socket says 172.21.15.105:55376) 2024-09-21T01:44:37.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.849+0000 7f92550f6640 1 -- 172.21.15.105:0/2149734158 learned_addr learned my addr 172.21.15.105:0/2149734158 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:37.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.849+0000 7fb16cff9640 1 -- 172.21.15.105:0/1041121881 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command([{prefix=config set, name=mds_debug_frag}] v 0) v1 -- 0x7fb1640055c0 con 0x7fb18008caa0 2024-09-21T01:44:37.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.850+0000 7f92550f6640 1 -- 172.21.15.105:0/2149734158 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f9250093080 msgr2=0x7f9250125730 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.850+0000 7f92550f6640 1 --2- 172.21.15.105:0/2149734158 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f9250093080 0x7f9250125730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.850+0000 7f92550f6640 1 -- 172.21.15.105:0/2149734158 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f92501295e0 msgr2=0x7f9250125c70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.850+0000 7f92550f6640 1 --2- 172.21.15.105:0/2149734158 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f92501295e0 0x7f9250125c70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.850+0000 7f92550f6640 1 -- 172.21.15.105:0/2149734158 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f924c009560 con 0x7f925008c200 2024-09-21T01:44:37.852 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.850+0000 7f92550f6640 1 --2- 172.21.15.105:0/2149734158 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f925008c200 0x7f92501384b0 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f924c00e9c0 tx=0x7f924c00ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:37.852 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.851+0000 7f9245ffb640 1 -- 172.21.15.105:0/2149734158 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f924c00cde0 con 0x7f925008c200 2024-09-21T01:44:37.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.851+0000 7f92560f8640 1 -- 172.21.15.105:0/2149734158 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9240009490 con 0x7f925008c200 2024-09-21T01:44:37.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.851+0000 7f92560f8640 1 -- 172.21.15.105:0/2149734158 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9250126390 con 0x7f925008c200 2024-09-21T01:44:37.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.853+0000 7f9245ffb640 1 -- 172.21.15.105:0/2149734158 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f924c004020 con 0x7f925008c200 2024-09-21T01:44:37.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.853+0000 7f9245ffb640 1 -- 172.21.15.105:0/2149734158 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f924c004410 con 0x7f925008c200 2024-09-21T01:44:37.854 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.854+0000 7f9245ffb640 1 -- 172.21.15.105:0/2149734158 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f924c005020 con 0x7f925008c200 2024-09-21T01:44:37.854 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.854+0000 7f9245ffb640 1 -- 172.21.15.105:0/2149734158 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f924c014070 con 0x7f925008c200 2024-09-21T01:44:37.855 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.854+0000 7f92560f8640 1 -- 172.21.15.105:0/2149734158 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f925006d6e0 con 0x7f925008c200 2024-09-21T01:44:37.859 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.859+0000 7f9245ffb640 1 -- 172.21.15.105:0/2149734158 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f924c0047c0 con 0x7f925008c200 2024-09-21T01:44:37.863 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.862+0000 7fb16effd640 1 -- 172.21.15.105:0/1041121881 <== mon.1 v2:172.21.15.105:3301/0 8 ==== mon_command_ack([{prefix=config set, name=mds_debug_frag}]=0 v9)=0 v9) v1 ==== 115+0+0 (secure 0 0 0) 0x7fb17002a0b0 con 0x7fb18008caa0 2024-09-21T01:44:37.864 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.863+0000 7fb184d33640 1 -- 172.21.15.105:0/1041121881 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb18008caa0 msgr2=0x7fb18007b6d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.864 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.863+0000 7fb184d33640 1 --2- 172.21.15.105:0/1041121881 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb18008caa0 0x7fb18007b6d0 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7fb17000d780 tx=0x7fb17000dc40 comp rx=0 tx=0).stop 2024-09-21T01:44:37.864 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.864+0000 7fb184d33640 1 -- 172.21.15.105:0/1041121881 shutdown_connections 2024-09-21T01:44:37.864 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.864+0000 7fb184d33640 1 --2- 172.21.15.105:0/1041121881 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb18007bc10 0x7fb1801d44a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.864 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.864+0000 7fb184d33640 1 --2- 172.21.15.105:0/1041121881 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb18008caa0 0x7fb18007b6d0 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.864 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.864+0000 7fb184d33640 1 --2- 172.21.15.105:0/1041121881 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb18006d030 0x7fb18007b190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.865 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.864+0000 7fb184d33640 1 -- 172.21.15.105:0/1041121881 >> 172.21.15.105:0/1041121881 conn(0x7fb180098b10 msgr2=0x7fb18006dea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:37.865 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.865+0000 7fb184d33640 1 -- 172.21.15.105:0/1041121881 shutdown_connections 2024-09-21T01:44:37.865 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.865+0000 7fb184d33640 1 -- 172.21.15.105:0/1041121881 wait complete. 2024-09-21T01:44:37.888 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.887+0000 7feeb3791640 1 -- 172.21.15.105:0/3319256422 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7feea400e2b0 msgr2=0x7feea40d0e10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.888 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.887+0000 7feeb3791640 1 --2- 172.21.15.105:0/3319256422 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7feea400e2b0 0x7feea40d0e10 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7feeac068190 tx=0x7feeac0945b0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.888 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.887+0000 7feeb3791640 1 -- 172.21.15.105:0/3319256422 shutdown_connections 2024-09-21T01:44:37.888 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.887+0000 7feeb3791640 1 --2- 172.21.15.105:0/3319256422 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feea40cdb80 0x7feea40d1350 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.888 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.887+0000 7feeb3791640 1 --2- 172.21.15.105:0/3319256422 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7feea400e2b0 0x7feea40d0e10 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.888 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.887+0000 7feeb3791640 1 --2- 172.21.15.105:0/3319256422 >> v2:172.21.15.105:3300/0 conn(0x7feea40ce340 0x7feea400dd70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.888 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.887+0000 7feeb3791640 1 -- 172.21.15.105:0/3319256422 >> 172.21.15.105:0/3319256422 conn(0x7feea4005aa0 msgr2=0x7feea4005eb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:37.894 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.888+0000 7feeb3791640 1 -- 172.21.15.105:0/3319256422 shutdown_connections 2024-09-21T01:44:37.894 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.888+0000 7feeb3791640 1 -- 172.21.15.105:0/3319256422 wait complete. 2024-09-21T01:44:37.894 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.889+0000 7feeb3791640 1 Processor -- start 2024-09-21T01:44:37.894 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.889+0000 7feeb3791640 1 -- start start 2024-09-21T01:44:37.894 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.889+0000 7feeb3791640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feea40cdb80 0x7feea4012530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.894 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.889+0000 7feeb3791640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feea40ce340 0x7feea4012a70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.895 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.889+0000 7feeb3791640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7feea4012fb0 0x7feea4013440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.895 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.889+0000 7feeb3791640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feea40d2c60 con 0x7feea40ce340 2024-09-21T01:44:37.895 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.889+0000 7feeb3791640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7feea40d3680 con 0x7feea40cdb80 2024-09-21T01:44:37.895 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.889+0000 7feeb3791640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7feea40d7df0 con 0x7feea4012fb0 2024-09-21T01:44:37.895 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.890+0000 7feeb152d640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feea40cdb80 0x7feea4012530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:37.895 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.890+0000 7feeb152d640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feea40cdb80 0x7feea4012530 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:50228/0 (socket says 172.21.15.105:50228) 2024-09-21T01:44:37.895 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.890+0000 7feeb152d640 1 -- 172.21.15.105:0/270538509 learned_addr learned my addr 172.21.15.105:0/270538509 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:37.895 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.890+0000 7feeb0d2c640 1 --2- 172.21.15.105:0/270538509 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feea40ce340 0x7feea4012a70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:37.895 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.891+0000 7feeb152d640 1 -- 172.21.15.105:0/270538509 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7feea4012fb0 msgr2=0x7feea4013440 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:44:37.895 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.891+0000 7feeb152d640 1 --2- 172.21.15.105:0/270538509 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7feea4012fb0 0x7feea4013440 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.895 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.891+0000 7feeb152d640 1 -- 172.21.15.105:0/270538509 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feea40ce340 msgr2=0x7feea4012a70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.895 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.891+0000 7feeb152d640 1 --2- 172.21.15.105:0/270538509 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feea40ce340 0x7feea4012a70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.895 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.891+0000 7feeb152d640 1 -- 172.21.15.105:0/270538509 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feea800c490 con 0x7feea40cdb80 2024-09-21T01:44:37.896 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.891+0000 7feeb0d2c640 1 --2- 172.21.15.105:0/270538509 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feea40ce340 0x7feea4012a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:37.896 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.891+0000 7feeb152d640 1 --2- 172.21.15.105:0/270538509 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feea40cdb80 0x7feea4012530 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7feea800fea0 tx=0x7feea800ff90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:37.896 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.893+0000 7f33e120a640 1 -- 172.21.15.105:0/2004126008 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f33d40b4150 msgr2=0x7f33d40bf5f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.896 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.893+0000 7f33e120a640 1 --2- 172.21.15.105:0/2004126008 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f33d40b4150 0x7f33d40bf5f0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f33dc0639a0 tx=0x7f33dc0a3400 comp rx=0 tx=0).stop 2024-09-21T01:44:37.896 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.893+0000 7feea1ffb640 1 -- 172.21.15.105:0/270538509 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feea800ce00 con 0x7feea40cdb80 2024-09-21T01:44:37.897 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.893+0000 7f33e120a640 1 -- 172.21.15.105:0/2004126008 shutdown_connections 2024-09-21T01:44:37.897 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.893+0000 7f33e120a640 1 --2- 172.21.15.105:0/2004126008 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f33d40b4150 0x7f33d40bf5f0 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.897 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.893+0000 7f33e120a640 1 --2- 172.21.15.105:0/2004126008 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f33d40b1d60 0x7f33d40bd0a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.898 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.893+0000 7f33e120a640 1 -- 172.21.15.105:0/2004126008 >> 172.21.15.105:0/2004126008 conn(0x7f33d4005af0 msgr2=0x7f33d4005f00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:37.898 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.893+0000 7f33e120a640 1 -- 172.21.15.105:0/2004126008 shutdown_connections 2024-09-21T01:44:37.898 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.893+0000 7f33e120a640 1 -- 172.21.15.105:0/2004126008 wait complete. 2024-09-21T01:44:37.898 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.895+0000 7f33e120a640 1 Processor -- start 2024-09-21T01:44:37.898 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.895+0000 7f33e120a640 1 -- start start 2024-09-21T01:44:37.898 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.895+0000 7f33e120a640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f33d40b1d60 0x7f33d40beba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.898 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.893+0000 7feeb3791640 1 -- 172.21.15.105:0/270538509 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7feeac06b630 con 0x7feea40cdb80 2024-09-21T01:44:37.898 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.894+0000 7feeb3791640 1 -- 172.21.15.105:0/270538509 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7feea40d3f10 con 0x7feea40cdb80 2024-09-21T01:44:37.898 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.896+0000 7feea1ffb640 1 -- 172.21.15.105:0/270538509 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7feea8004110 con 0x7feea40cdb80 2024-09-21T01:44:37.899 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.895+0000 7f33e120a640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f33d40a9e00 0x7f33d40aa240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.899 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.895+0000 7f33e120a640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f33d40aa780 0x7f33d40aac00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.899 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.895+0000 7f33e120a640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f33d40ab140 con 0x7f33d40a9e00 2024-09-21T01:44:37.899 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.895+0000 7f33e120a640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f33d40ace80 con 0x7f33d40b1d60 2024-09-21T01:44:37.899 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.895+0000 7f33e120a640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f33d401e710 con 0x7f33d40aa780 2024-09-21T01:44:37.900 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.896+0000 7feea1ffb640 1 -- 172.21.15.105:0/270538509 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7feea80044e0 con 0x7feea40cdb80 2024-09-21T01:44:37.900 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.896+0000 7feea1ffb640 1 -- 172.21.15.105:0/270538509 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7feea8005320 con 0x7feea40cdb80 2024-09-21T01:44:37.900 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.896+0000 7f33dbfff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f33d40b1d60 0x7f33d40beba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:37.900 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.896+0000 7f33db7fe640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f33d40a9e00 0x7f33d40aa240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:37.900 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.896+0000 7f33dbfff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f33d40b1d60 0x7f33d40beba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:50266/0 (socket says 172.21.15.105:50266) 2024-09-21T01:44:37.900 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.896+0000 7feea1ffb640 1 -- 172.21.15.105:0/270538509 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7feea80036c0 con 0x7feea40cdb80 2024-09-21T01:44:37.901 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.896+0000 7f33dbfff640 1 -- 172.21.15.105:0/3661563747 learned_addr learned my addr 172.21.15.105:0/3661563747 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:37.901 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.896+0000 7f33db7fe640 1 -- 172.21.15.105:0/3661563747 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f33d40aa780 msgr2=0x7f33d40aac00 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:44:37.901 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.896+0000 7f33db7fe640 1 --2- 172.21.15.105:0/3661563747 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f33d40aa780 0x7f33d40aac00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.902 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.896+0000 7f33db7fe640 1 -- 172.21.15.105:0/3661563747 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f33d40b1d60 msgr2=0x7f33d40beba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.902 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.898+0000 7feeb3791640 1 -- 172.21.15.105:0/270538509 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fee8c005250 con 0x7feea40cdb80 2024-09-21T01:44:37.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.901+0000 7f33db7fe640 1 --2- 172.21.15.105:0/3661563747 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f33d40b1d60 0x7f33d40beba0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.901+0000 7f33db7fe640 1 -- 172.21.15.105:0/3661563747 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f33dc073160 con 0x7f33d40a9e00 2024-09-21T01:44:37.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.901+0000 7f33db7fe640 1 --2- 172.21.15.105:0/3661563747 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f33d40a9e00 0x7f33d40aa240 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f33c800ca30 tx=0x7f33c800cef0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:37.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.901+0000 7f33d8ff9640 1 -- 172.21.15.105:0/3661563747 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f33c8007ac0 con 0x7f33d40a9e00 2024-09-21T01:44:37.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.901+0000 7f33d8ff9640 1 -- 172.21.15.105:0/3661563747 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f33c8007d10 con 0x7f33d40a9e00 2024-09-21T01:44:37.904 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.901+0000 7f33d8ff9640 1 -- 172.21.15.105:0/3661563747 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f33c800f6b0 con 0x7f33d40a9e00 2024-09-21T01:44:37.904 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.901+0000 7feea1ffb640 1 -- 172.21.15.105:0/270538509 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7feea80047c0 con 0x7feea40cdb80 2024-09-21T01:44:37.904 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.902+0000 7f24fb6e5640 1 -- 172.21.15.105:0/2027426740 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f24f406d030 msgr2=0x7f24f408c730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.904 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.902+0000 7f24fb6e5640 1 --2- 172.21.15.105:0/2027426740 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f24f406d030 0x7f24f408c730 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f24ec005fd0 tx=0x7f24ec02abe0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.904 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.903+0000 7f24fb6e5640 1 -- 172.21.15.105:0/2027426740 shutdown_connections 2024-09-21T01:44:37.904 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.903+0000 7f24fb6e5640 1 --2- 172.21.15.105:0/2027426740 >> v2:172.21.15.105:3300/0 conn(0x7f24f4094f50 0x7f24f4097410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.903+0000 7f24fb6e5640 1 --2- 172.21.15.105:0/2027426740 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f24f406d030 0x7f24f408c730 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.903+0000 7f24fb6e5640 1 -- 172.21.15.105:0/2027426740 >> 172.21.15.105:0/2027426740 conn(0x7f24f4098b10 msgr2=0x7f24f409a610 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:37.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.903+0000 7f33dbfff640 1 --2- 172.21.15.105:0/3661563747 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f33d40b1d60 0x7f33d40beba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:37.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.903+0000 7f33e120a640 1 -- 172.21.15.105:0/3661563747 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f33d40bf0e0 con 0x7f33d40a9e00 2024-09-21T01:44:37.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.903+0000 7f33e120a640 1 -- 172.21.15.105:0/3661563747 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f33d40ad670 con 0x7f33d40a9e00 2024-09-21T01:44:37.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.903+0000 7f24fb6e5640 1 -- 172.21.15.105:0/2027426740 shutdown_connections 2024-09-21T01:44:37.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.904+0000 7f24fb6e5640 1 -- 172.21.15.105:0/2027426740 wait complete. 2024-09-21T01:44:37.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.905+0000 7f24fb6e5640 1 Processor -- start 2024-09-21T01:44:37.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.905+0000 7f24fb6e5640 1 -- start start 2024-09-21T01:44:37.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.905+0000 7f24fb6e5640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f24f4094f50 0x7f24f4093f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.905+0000 7f24fb6e5640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f24f409ad20 0x7f24f40944a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.905+0000 7f24fb6e5640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f24f40949e0 0x7f24f4079a60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.905+0000 7f24fb6e5640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f24f407be20 con 0x7f24f409ad20 2024-09-21T01:44:37.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.905+0000 7f24fb6e5640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f24f407c720 con 0x7f24f40949e0 2024-09-21T01:44:37.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.905+0000 7f24fb6e5640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f24f407de80 con 0x7f24f4094f50 2024-09-21T01:44:37.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.906+0000 7f24f9481640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f24f4094f50 0x7f24f4093f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:37.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.906+0000 7f24f9481640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f24f4094f50 0x7f24f4093f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:45504/0 (socket says 172.21.15.105:45504) 2024-09-21T01:44:37.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.906+0000 7f24f9481640 1 -- 172.21.15.105:0/1798968300 learned_addr learned my addr 172.21.15.105:0/1798968300 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:37.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.906+0000 7f24f8c80640 1 --2- 172.21.15.105:0/1798968300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f24f409ad20 0x7f24f40944a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:37.908 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.906+0000 7f24f9481640 1 -- 172.21.15.105:0/1798968300 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f24f40949e0 msgr2=0x7f24f4079a60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.908 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.906+0000 7f24f9481640 1 --2- 172.21.15.105:0/1798968300 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f24f40949e0 0x7f24f4079a60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.908 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.906+0000 7f24f9481640 1 -- 172.21.15.105:0/1798968300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f24f409ad20 msgr2=0x7f24f40944a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.908 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.906+0000 7f24f9481640 1 --2- 172.21.15.105:0/1798968300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f24f409ad20 0x7f24f40944a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.908 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.906+0000 7f24f9481640 1 -- 172.21.15.105:0/1798968300 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f24ec009900 con 0x7f24f4094f50 2024-09-21T01:44:37.908 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.907+0000 7f24f9481640 1 --2- 172.21.15.105:0/1798968300 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f24f4094f50 0x7f24f4093f60 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f24f000b450 tx=0x7f24f000b910 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:37.908 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.907+0000 7f24e9ffb640 1 -- 172.21.15.105:0/1798968300 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f24f0011830 con 0x7f24f4094f50 2024-09-21T01:44:37.910 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.907+0000 7f24fb6e5640 1 -- 172.21.15.105:0/1798968300 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f24f4072200 con 0x7f24f4094f50 2024-09-21T01:44:37.910 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.908+0000 7f24fb6e5640 1 -- 172.21.15.105:0/1798968300 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f24f4079fa0 con 0x7f24f4094f50 2024-09-21T01:44:37.910 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.909+0000 7f24e9ffb640 1 -- 172.21.15.105:0/1798968300 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f24f0011aa0 con 0x7f24f4094f50 2024-09-21T01:44:37.910 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.909+0000 7f24e9ffb640 1 -- 172.21.15.105:0/1798968300 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f24f0010380 con 0x7f24f4094f50 2024-09-21T01:44:37.911 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.911+0000 7f24e9ffb640 1 -- 172.21.15.105:0/1798968300 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f24f0010860 con 0x7f24f4094f50 2024-09-21T01:44:37.912 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.912+0000 7f33e120a640 1 -- 172.21.15.105:0/3661563747 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f33bc005250 con 0x7f33d40a9e00 2024-09-21T01:44:37.912 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.912+0000 7f24e9ffb640 1 -- 172.21.15.105:0/1798968300 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f24f001a9e0 con 0x7f24f4094f50 2024-09-21T01:44:37.913 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.912+0000 7f24fb6e5640 1 -- 172.21.15.105:0/1798968300 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f24d4005250 con 0x7f24f4094f50 2024-09-21T01:44:37.916 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.916+0000 7f24e9ffb640 1 -- 172.21.15.105:0/1798968300 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f24f0039e30 con 0x7f24f4094f50 2024-09-21T01:44:37.922 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.921+0000 7f33d8ff9640 1 -- 172.21.15.105:0/3661563747 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f33c8005020 con 0x7f33d40a9e00 2024-09-21T01:44:37.922 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.922+0000 7f33d8ff9640 1 -- 172.21.15.105:0/3661563747 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f33c80047a0 con 0x7f33d40a9e00 2024-09-21T01:44:37.922 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.922+0000 7f33d8ff9640 1 -- 172.21.15.105:0/3661563747 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f33c8004a70 con 0x7f33d40a9e00 2024-09-21T01:44:37.923 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:37.923+0000 7f43c91dd040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T01:44:37.924 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.923+0000 7fadd9c2d640 1 -- 172.21.15.105:0/325281363 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fadcc00da90 msgr2=0x7fadcc00de70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.924 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.923+0000 7fadd9c2d640 1 --2- 172.21.15.105:0/325281363 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fadcc00da90 0x7fadcc00de70 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7fadc000dd10 tx=0x7fadc0032a30 comp rx=0 tx=0).stop 2024-09-21T01:44:37.925 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.925+0000 7f1c65391640 1 -- 172.21.15.105:0/1023999749 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1c6006c9a0 msgr2=0x7f1c6006ce00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.925 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.925+0000 7f1c65391640 1 --2- 172.21.15.105:0/1023999749 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1c6006c9a0 0x7f1c6006ce00 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f1c50005fd0 tx=0x7f1c5002b670 comp rx=0 tx=0).stop 2024-09-21T01:44:37.925 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.925+0000 7f1c65391640 1 -- 172.21.15.105:0/1023999749 shutdown_connections 2024-09-21T01:44:37.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.925+0000 7f1c65391640 1 --2- 172.21.15.105:0/1023999749 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1c6006c9a0 0x7f1c6006ce00 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.925+0000 7f1c65391640 1 --2- 172.21.15.105:0/1023999749 >> v2:172.21.15.105:3300/0 conn(0x7f1c60128be0 0x7f1c6006c460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.925+0000 7f1c65391640 1 --2- 172.21.15.105:0/1023999749 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f1c601282c0 0x7f1c601286a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.925+0000 7f1c65391640 1 -- 172.21.15.105:0/1023999749 >> 172.21.15.105:0/1023999749 conn(0x7f1c60099470 msgr2=0x7f1c60099880 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:37.937 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.929+0000 7fadd9c2d640 1 -- 172.21.15.105:0/325281363 shutdown_connections 2024-09-21T01:44:37.937 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.929+0000 7fadd9c2d640 1 --2- 172.21.15.105:0/325281363 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fadcc00da90 0x7fadcc00de70 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.937 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.929+0000 7fadd9c2d640 1 --2- 172.21.15.105:0/325281363 >> v2:172.21.15.105:3300/0 conn(0x7fadcc0afdd0 0x7fadcc0b01b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.937 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.929+0000 7fadd9c2d640 1 -- 172.21.15.105:0/325281363 >> 172.21.15.105:0/325281363 conn(0x7fadcc005af0 msgr2=0x7fadcc005f00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:37.937 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.937+0000 7fadd9c2d640 1 -- 172.21.15.105:0/325281363 shutdown_connections 2024-09-21T01:44:37.938 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.937+0000 7fa146584640 1 -- 172.21.15.105:0/1500671650 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command([{prefix=config set, name=debug_ms}] v 0) v1 -- 0x7fa120001040 con 0x7fa14012c010 2024-09-21T01:44:37.938 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.938+0000 7fadd9c2d640 1 -- 172.21.15.105:0/325281363 wait complete. 2024-09-21T01:44:37.938 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.929+0000 7f1c65391640 1 -- 172.21.15.105:0/1023999749 shutdown_connections 2024-09-21T01:44:37.939 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.939+0000 7fadd9c2d640 1 Processor -- start 2024-09-21T01:44:37.941 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:37.939+0000 7fc75c220040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T01:44:37.941 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.939+0000 7fadd9c2d640 1 -- start start 2024-09-21T01:44:37.941 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.940+0000 7fadd9c2d640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fadcc00da90 0x7fadcc0b0a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.941 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.940+0000 7fadd9c2d640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fadcc00e3b0 0x7fadcc0b0f50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.941 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.940+0000 7fadd9c2d640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fadcc0afdd0 0x7fadcc0b1490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.941 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.940+0000 7fadd9c2d640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fadcc0b7cf0 con 0x7fadcc00e3b0 2024-09-21T01:44:37.941 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.940+0000 7fadd9c2d640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fadcc0ac5a0 con 0x7fadcc0afdd0 2024-09-21T01:44:37.941 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.941+0000 7fadd9c2d640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fadcc013ad0 con 0x7fadcc00da90 2024-09-21T01:44:37.942 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.938+0000 7f1c65391640 1 -- 172.21.15.105:0/1023999749 wait complete. 2024-09-21T01:44:37.942 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.939+0000 7f1c65391640 1 Processor -- start 2024-09-21T01:44:37.942 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.939+0000 7f1c65391640 1 -- start start 2024-09-21T01:44:37.943 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.941+0000 7fadd942c640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fadcc0afdd0 0x7fadcc0b1490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:37.943 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.941+0000 7fadd942c640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fadcc0afdd0 0x7fadcc0b1490 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:50304/0 (socket says 172.21.15.105:50304) 2024-09-21T01:44:37.943 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.941+0000 7fadd942c640 1 -- 172.21.15.105:0/677805864 learned_addr learned my addr 172.21.15.105:0/677805864 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:37.943 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.942+0000 7fadd942c640 1 -- 172.21.15.105:0/677805864 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fadcc00da90 msgr2=0x7fadcc0b0a10 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:44:37.943 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.942+0000 7fadd942c640 1 --2- 172.21.15.105:0/677805864 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fadcc00da90 0x7fadcc0b0a10 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.943 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.942+0000 7fadd942c640 1 -- 172.21.15.105:0/677805864 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fadcc00e3b0 msgr2=0x7fadcc0b0f50 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:44:37.943 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.942+0000 7fadd942c640 1 --2- 172.21.15.105:0/677805864 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fadcc00e3b0 0x7fadcc0b0f50 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.944 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.942+0000 7fadd942c640 1 -- 172.21.15.105:0/677805864 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fadc8009490 con 0x7fadcc0afdd0 2024-09-21T01:44:37.944 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.943+0000 7fadd942c640 1 --2- 172.21.15.105:0/677805864 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fadcc0afdd0 0x7fadcc0b1490 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7fadd40639a0 tx=0x7fadd40667b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:37.944 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.943+0000 7fadd17fa640 1 -- 172.21.15.105:0/677805864 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fadd4072150 con 0x7fadcc0afdd0 2024-09-21T01:44:37.944 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.943+0000 7fadd9c2d640 1 -- 172.21.15.105:0/677805864 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fadc000d970 con 0x7fadcc0afdd0 2024-09-21T01:44:37.944 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.944+0000 7fadd9c2d640 1 -- 172.21.15.105:0/677805864 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fadcc0acd30 con 0x7fadcc0afdd0 2024-09-21T01:44:37.945 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.944+0000 7fadd17fa640 1 -- 172.21.15.105:0/677805864 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fadd40722b0 con 0x7fadcc0afdd0 2024-09-21T01:44:37.945 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.944+0000 7fadd17fa640 1 -- 172.21.15.105:0/677805864 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fadd40673f0 con 0x7fadcc0afdd0 2024-09-21T01:44:37.945 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.945+0000 7fadd17fa640 1 -- 172.21.15.105:0/677805864 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7fadd4073380 con 0x7fadcc0afdd0 2024-09-21T01:44:37.945 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.945+0000 7fadd17fa640 1 -- 172.21.15.105:0/677805864 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7fadd4066ba0 con 0x7fadcc0afdd0 2024-09-21T01:44:37.956 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.940+0000 7f1c65391640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1c6006c9a0 0x7f1c6008e050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.957 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.940+0000 7f1c65391640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1c601282c0 0x7f1c6008e590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.957 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.940+0000 7f1c65391640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f1c60128be0 0x7f1c6008ead0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.957 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.940+0000 7f1c65391640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1c600977f0 con 0x7f1c601282c0 2024-09-21T01:44:37.957 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.940+0000 7f1c65391640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1c60098000 con 0x7f1c60128be0 2024-09-21T01:44:37.957 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.940+0000 7f1c65391640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f1c60071470 con 0x7f1c6006c9a0 2024-09-21T01:44:37.957 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.940+0000 7f1c5effd640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1c6006c9a0 0x7f1c6008e050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:37.957 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.940+0000 7f1c5effd640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1c6006c9a0 0x7f1c6008e050 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:45524/0 (socket says 172.21.15.105:45524) 2024-09-21T01:44:37.957 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.940+0000 7f1c5effd640 1 -- 172.21.15.105:0/4207177331 learned_addr learned my addr 172.21.15.105:0/4207177331 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:37.957 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.941+0000 7f1c5effd640 1 -- 172.21.15.105:0/4207177331 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f1c60128be0 msgr2=0x7f1c6008ead0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:44:37.957 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.941+0000 7f1c5effd640 1 --2- 172.21.15.105:0/4207177331 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f1c60128be0 0x7f1c6008ead0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.957 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.941+0000 7f1c5effd640 1 -- 172.21.15.105:0/4207177331 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1c601282c0 msgr2=0x7f1c6008e590 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:44:37.957 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.941+0000 7f1c5effd640 1 --2- 172.21.15.105:0/4207177331 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1c601282c0 0x7f1c6008e590 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.957 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.941+0000 7f1c5effd640 1 -- 172.21.15.105:0/4207177331 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1c54009490 con 0x7f1c6006c9a0 2024-09-21T01:44:37.958 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.941+0000 7f1c5effd640 1 --2- 172.21.15.105:0/4207177331 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1c6006c9a0 0x7f1c6008e050 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f1c5800cd10 tx=0x7f1c5800a620 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:37.958 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.955+0000 7f1c477fe640 1 -- 172.21.15.105:0/4207177331 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1c5800f650 con 0x7f1c6006c9a0 2024-09-21T01:44:37.958 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.955+0000 7f1c477fe640 1 -- 172.21.15.105:0/4207177331 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f1c5800f8a0 con 0x7f1c6006c9a0 2024-09-21T01:44:37.958 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.955+0000 7f1c477fe640 1 -- 172.21.15.105:0/4207177331 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1c5800fc90 con 0x7f1c6006c9a0 2024-09-21T01:44:37.958 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.956+0000 7fa13cff9640 1 -- 172.21.15.105:0/1500671650 <== mon.2 v2:172.21.15.105:3302/0 11 ==== mon_command_ack([{prefix=config set, name=debug_ms}]=0 v10)=0 v10) v1 ==== 106+0+0 (secure 0 0 0) 0x7fa138045020 con 0x7fa14012c010 2024-09-21T01:44:37.958 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.957+0000 7fadd9c2d640 1 -- 172.21.15.105:0/677805864 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fadb4005250 con 0x7fadcc0afdd0 2024-09-21T01:44:37.959 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.956+0000 7f1c65391640 1 -- 172.21.15.105:0/4207177331 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1c50009df0 con 0x7f1c6006c9a0 2024-09-21T01:44:37.959 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.956+0000 7f1c65391640 1 -- 172.21.15.105:0/4207177331 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1c6008f0d0 con 0x7f1c6006c9a0 2024-09-21T01:44:37.959 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.957+0000 7f1c477fe640 1 -- 172.21.15.105:0/4207177331 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f1c58018400 con 0x7f1c6006c9a0 2024-09-21T01:44:37.959 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.957+0000 7f1c477fe640 1 -- 172.21.15.105:0/4207177331 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f1c58021730 con 0x7f1c6006c9a0 2024-09-21T01:44:37.959 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.958+0000 7f1c65391640 1 -- 172.21.15.105:0/4207177331 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1c40005250 con 0x7f1c6006c9a0 2024-09-21T01:44:37.961 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.961+0000 7fa146584640 1 -- 172.21.15.105:0/1500671650 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa14012c010 msgr2=0x7fa140072070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.961 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.961+0000 7fa146584640 1 --2- 172.21.15.105:0/1500671650 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa14012c010 0x7fa140072070 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7fa13803bf30 tx=0x7fa138004a40 comp rx=0 tx=0).stop 2024-09-21T01:44:37.961 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.961+0000 7fa146584640 1 -- 172.21.15.105:0/1500671650 shutdown_connections 2024-09-21T01:44:37.961 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.961+0000 7fa146584640 1 --2- 172.21.15.105:0/1500671650 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa14012c010 0x7fa140072070 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.962 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.961+0000 7fa146584640 1 --2- 172.21.15.105:0/1500671650 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa14012b5f0 0x7fa140071b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.962 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.961+0000 7fa146584640 1 --2- 172.21.15.105:0/1500671650 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa14008c410 0x7fa1400715f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.962 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.961+0000 7fa146584640 1 -- 172.21.15.105:0/1500671650 >> 172.21.15.105:0/1500671650 conn(0x7fa14009aa30 msgr2=0x7fa140089820 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:37.962 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.961+0000 7fa146584640 1 -- 172.21.15.105:0/1500671650 shutdown_connections 2024-09-21T01:44:37.962 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.961+0000 7fa146584640 1 -- 172.21.15.105:0/1500671650 wait complete. 2024-09-21T01:44:37.962 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.962+0000 7fadd17fa640 1 -- 172.21.15.105:0/677805864 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7fadd4067850 con 0x7fadcc0afdd0 2024-09-21T01:44:37.963 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.962+0000 7f1c477fe640 1 -- 172.21.15.105:0/4207177331 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f1c5800fdf0 con 0x7f1c6006c9a0 2024-09-21T01:44:37.963 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.962+0000 7f98ff36c640 1 -- 172.21.15.105:0/1364319295 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f98f000d8b0 msgr2=0x7f98f000dc90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.963 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.962+0000 7f98ff36c640 1 --2- 172.21.15.105:0/1364319295 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f98f000d8b0 0x7f98f000dc90 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f98f8068190 tx=0x7f98f80a48d0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.966 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.966+0000 7f98ff36c640 1 -- 172.21.15.105:0/1364319295 shutdown_connections 2024-09-21T01:44:37.966 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.966+0000 7f98ff36c640 1 --2- 172.21.15.105:0/1364319295 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f98f000e650 0x7f98f00b6430 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.966 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.966+0000 7f98ff36c640 1 --2- 172.21.15.105:0/1364319295 >> v2:172.21.15.105:3300/0 conn(0x7f98f000e1d0 0x7f98f00b3ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.966 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.966+0000 7f98ff36c640 1 --2- 172.21.15.105:0/1364319295 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f98f000d8b0 0x7f98f000dc90 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.966 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.966+0000 7f98ff36c640 1 -- 172.21.15.105:0/1364319295 >> 172.21.15.105:0/1364319295 conn(0x7f98f0006710 msgr2=0x7f98f0006b20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:37.968 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.966+0000 7f98ff36c640 1 -- 172.21.15.105:0/1364319295 shutdown_connections 2024-09-21T01:44:37.969 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.968+0000 7f98ff36c640 1 -- 172.21.15.105:0/1364319295 wait complete. 2024-09-21T01:44:37.969 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.969+0000 7f98ff36c640 1 Processor -- start 2024-09-21T01:44:37.969 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.969+0000 7f98ff36c640 1 -- start start 2024-09-21T01:44:37.970 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.970+0000 7f98ff36c640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f98f000d8b0 0x7f98f0142190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.971 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.971+0000 7f98fe36a640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f98f000d8b0 0x7f98f0142190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:37.971 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.970+0000 7f73b2a17640 1 -- 172.21.15.105:0/3085637461 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f73ac06cae0 msgr2=0x7f73ac06cf20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.971 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.971+0000 7f98fe36a640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f98f000d8b0 0x7f98f0142190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:45548/0 (socket says 172.21.15.105:45548) 2024-09-21T01:44:37.972 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.971+0000 7f98fe36a640 1 -- 172.21.15.105:0/1459717353 learned_addr learned my addr 172.21.15.105:0/1459717353 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:37.972 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.971+0000 7f98ff36c640 1 --2- 172.21.15.105:0/1459717353 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f98f000e1d0 0x7f98f01426d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.972 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.970+0000 7f73b2a17640 1 --2- 172.21.15.105:0/3085637461 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f73ac06cae0 0x7f73ac06cf20 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f73a4005fd0 tx=0x7f73a402b000 comp rx=0 tx=0).stop 2024-09-21T01:44:37.972 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.971+0000 7f98ff36c640 1 --2- 172.21.15.105:0/1459717353 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f98f000e650 0x7f98f0142c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.972 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.971+0000 7f98ff36c640 1 -- 172.21.15.105:0/1459717353 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f98f0143d40 con 0x7f98f000e1d0 2024-09-21T01:44:37.973 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.971+0000 7f73b2a17640 1 -- 172.21.15.105:0/3085637461 shutdown_connections 2024-09-21T01:44:37.973 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.971+0000 7f73b2a17640 1 --2- 172.21.15.105:0/3085637461 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f73ac094090 0x7f73ac0964f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.973 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.971+0000 7f73b2a17640 1 --2- 172.21.15.105:0/3085637461 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f73ac06cae0 0x7f73ac06cf20 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.973 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.971+0000 7f73b2a17640 1 --2- 172.21.15.105:0/3085637461 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f73ac088910 0x7f73ac06c5a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.973 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.971+0000 7f73b2a17640 1 -- 172.21.15.105:0/3085637461 >> 172.21.15.105:0/3085637461 conn(0x7f73ac09a610 msgr2=0x7f73ac09aa20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:37.974 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.971+0000 7f98ff36c640 1 -- 172.21.15.105:0/1459717353 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f98f0016170 con 0x7f98f000e650 2024-09-21T01:44:37.974 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.971+0000 7f98ff36c640 1 -- 172.21.15.105:0/1459717353 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f98f00139e0 con 0x7f98f000d8b0 2024-09-21T01:44:37.974 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.972+0000 7f98fe36a640 1 -- 172.21.15.105:0/1459717353 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f98f000e650 msgr2=0x7f98f0142c10 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:44:37.975 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.972+0000 7f98fe36a640 1 --2- 172.21.15.105:0/1459717353 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f98f000e650 0x7f98f0142c10 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.975 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.972+0000 7f98fe36a640 1 -- 172.21.15.105:0/1459717353 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f98f000e1d0 msgr2=0x7f98f01426d0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:44:37.975 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.972+0000 7f98fe36a640 1 --2- 172.21.15.105:0/1459717353 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f98f000e1d0 0x7f98f01426d0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.975 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.972+0000 7f98fe36a640 1 -- 172.21.15.105:0/1459717353 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f98f806b630 con 0x7f98f000d8b0 2024-09-21T01:44:37.975 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.972+0000 7f98fe36a640 1 --2- 172.21.15.105:0/1459717353 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f98f000d8b0 0x7f98f0142190 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f98f8068e50 tx=0x7f98f80aad80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:37.975 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.973+0000 7f98eeffd640 1 -- 172.21.15.105:0/1459717353 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f98f80af070 con 0x7f98f000d8b0 2024-09-21T01:44:37.975 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.973+0000 7f98ff36c640 1 -- 172.21.15.105:0/1459717353 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f98f0015ec0 con 0x7f98f000d8b0 2024-09-21T01:44:37.975 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.973+0000 7f98ff36c640 1 -- 172.21.15.105:0/1459717353 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f98f00b5790 con 0x7f98f000d8b0 2024-09-21T01:44:37.975 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.973+0000 7f0b0a7fc640 1 -- 172.21.15.105:0/672714383 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command([{prefix=config set, name=mds_debug_scatterstat}] v 0) v1 -- 0x7f0b2406af40 con 0x7f0b241292f0 2024-09-21T01:44:37.986 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.971+0000 7f73b2a17640 1 -- 172.21.15.105:0/3085637461 shutdown_connections 2024-09-21T01:44:37.986 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.971+0000 7f73b2a17640 1 -- 172.21.15.105:0/3085637461 wait complete. 2024-09-21T01:44:37.986 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.972+0000 7f73b2a17640 1 Processor -- start 2024-09-21T01:44:37.986 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.972+0000 7f73b2a17640 1 -- start start 2024-09-21T01:44:37.986 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.973+0000 7f73b2a17640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f73ac088910 0x7f73ac1d0700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.986 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.973+0000 7f73b2a17640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f73ac094090 0x7f73ac1d0c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.987 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.973+0000 7f73b2a17640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f73ac08f320 0x7f73ac08f7b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.987 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.973+0000 7f73b2a17640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f73ac092980 con 0x7f73ac088910 2024-09-21T01:44:37.987 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.973+0000 7f73b2a17640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f73ac0926c0 con 0x7f73ac094090 2024-09-21T01:44:37.987 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.973+0000 7f73b2a17640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f73ac092bd0 con 0x7f73ac08f320 2024-09-21T01:44:37.987 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.983+0000 7f73b1214640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f73ac094090 0x7f73ac1d0c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:37.987 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.983+0000 7f73b1214640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f73ac094090 0x7f73ac1d0c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:50322/0 (socket says 172.21.15.105:50322) 2024-09-21T01:44:37.987 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.983+0000 7f73b1214640 1 -- 172.21.15.105:0/3498580097 learned_addr learned my addr 172.21.15.105:0/3498580097 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:37.987 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.983+0000 7f73b1214640 1 -- 172.21.15.105:0/3498580097 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f73ac08f320 msgr2=0x7f73ac08f7b0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:44:37.987 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.983+0000 7f73b1214640 1 --2- 172.21.15.105:0/3498580097 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f73ac08f320 0x7f73ac08f7b0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.987 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.983+0000 7f73b1214640 1 -- 172.21.15.105:0/3498580097 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f73ac088910 msgr2=0x7f73ac1d0700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.987 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.984+0000 7f73b1214640 1 --2- 172.21.15.105:0/3498580097 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f73ac088910 0x7f73ac1d0700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.987 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.984+0000 7f73b1214640 1 -- 172.21.15.105:0/3498580097 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f73a4009900 con 0x7f73ac094090 2024-09-21T01:44:37.987 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.984+0000 7f73b1214640 1 --2- 172.21.15.105:0/3498580097 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f73ac094090 0x7f73ac1d0c40 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f73a4009e50 tx=0x7f73a402cba0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:37.988 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.985+0000 7f73a27fc640 1 -- 172.21.15.105:0/3498580097 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f73a4033880 con 0x7f73ac094090 2024-09-21T01:44:37.988 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.986+0000 7f73b2a17640 1 -- 172.21.15.105:0/3498580097 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f73ac1d1180 con 0x7f73ac094090 2024-09-21T01:44:37.989 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.986+0000 7f73b2a17640 1 -- 172.21.15.105:0/3498580097 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f73ac08fcf0 con 0x7f73ac094090 2024-09-21T01:44:37.989 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.988+0000 7f73a27fc640 1 -- 172.21.15.105:0/3498580097 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f73a4033ad0 con 0x7f73ac094090 2024-09-21T01:44:37.989 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.988+0000 7f73a27fc640 1 -- 172.21.15.105:0/3498580097 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f73a4033ea0 con 0x7f73ac094090 2024-09-21T01:44:37.989 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.988+0000 7f73a27fc640 1 -- 172.21.15.105:0/3498580097 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f73a400a040 con 0x7f73ac094090 2024-09-21T01:44:37.989 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.988+0000 7f73a27fc640 1 -- 172.21.15.105:0/3498580097 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f73a403f860 con 0x7f73ac094090 2024-09-21T01:44:37.989 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.988+0000 7f0b20ff9640 1 -- 172.21.15.105:0/672714383 <== mon.2 v2:172.21.15.105:3302/0 8 ==== mon_command_ack([{prefix=config set, name=mds_debug_scatterstat}]=0 v11)=0 v11) v1 ==== 122+0+0 (secure 0 0 0) 0x7f0b1c003d30 con 0x7f0b241292f0 2024-09-21T01:44:37.989 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.989+0000 7f73b2a17640 1 -- 172.21.15.105:0/3498580097 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7394005250 con 0x7f73ac094090 2024-09-21T01:44:37.990 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.977+0000 7fbc2533a640 1 -- 172.21.15.105:0/2048684796 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc1800e300 msgr2=0x7fbc180d0e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.991 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.977+0000 7fbc2533a640 1 --2- 172.21.15.105:0/2048684796 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc1800e300 0x7fbc180d0e60 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7fbc20068190 tx=0x7fbc20094590 comp rx=0 tx=0).stop 2024-09-21T01:44:37.991 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.977+0000 7fbc2533a640 1 -- 172.21.15.105:0/2048684796 shutdown_connections 2024-09-21T01:44:37.991 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.977+0000 7fbc2533a640 1 --2- 172.21.15.105:0/2048684796 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbc180cdbd0 0x7fbc180d13a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.991 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.977+0000 7fbc2533a640 1 --2- 172.21.15.105:0/2048684796 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc1800e300 0x7fbc180d0e60 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.991 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.977+0000 7fbc2533a640 1 --2- 172.21.15.105:0/2048684796 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbc180ce390 0x7fbc1800ddc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.991 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.977+0000 7fbc2533a640 1 -- 172.21.15.105:0/2048684796 >> 172.21.15.105:0/2048684796 conn(0x7fbc18005af0 msgr2=0x7fbc18005f00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:37.991 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.977+0000 7fbc2533a640 1 -- 172.21.15.105:0/2048684796 shutdown_connections 2024-09-21T01:44:37.991 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.977+0000 7fbc2533a640 1 -- 172.21.15.105:0/2048684796 wait complete. 2024-09-21T01:44:37.991 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.978+0000 7fbc2533a640 1 Processor -- start 2024-09-21T01:44:37.991 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.989+0000 7fbc2533a640 1 -- start start 2024-09-21T01:44:37.991 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.989+0000 7fbc2533a640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbc180cdbd0 0x7fbc18012890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.991 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.989+0000 7fbc2533a640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc180ce390 0x7fbc18012dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.992 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.989+0000 7fbc2533a640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbc18013310 0x7fbc180e4010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:37.992 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.990+0000 7fbc2533a640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbc180d2be0 con 0x7fbc180ce390 2024-09-21T01:44:37.992 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.990+0000 7fbc2533a640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbc180d2dc0 con 0x7fbc180cdbd0 2024-09-21T01:44:37.992 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.990+0000 7fbc2533a640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fbc180cb650 con 0x7fbc18013310 2024-09-21T01:44:37.992 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.990+0000 7fbc1ffff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbc180cdbd0 0x7fbc18012890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:37.992 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.990+0000 7fbc1ffff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbc180cdbd0 0x7fbc18012890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:50324/0 (socket says 172.21.15.105:50324) 2024-09-21T01:44:37.992 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.990+0000 7fbc1ffff640 1 -- 172.21.15.105:0/3917040209 learned_addr learned my addr 172.21.15.105:0/3917040209 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:37.992 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.990+0000 7fbc1f7fe640 1 --2- 172.21.15.105:0/3917040209 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc180ce390 0x7fbc18012dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:37.992 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.990+0000 7fbc24b39640 1 --2- 172.21.15.105:0/3917040209 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbc18013310 0x7fbc180e4010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:37.993 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.991+0000 7fbc1ffff640 1 -- 172.21.15.105:0/3917040209 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbc18013310 msgr2=0x7fbc180e4010 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.993 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.991+0000 7fbc1ffff640 1 --2- 172.21.15.105:0/3917040209 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbc18013310 0x7fbc180e4010 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.993 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.991+0000 7fbc1ffff640 1 -- 172.21.15.105:0/3917040209 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc180ce390 msgr2=0x7fbc18012dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.993 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.991+0000 7fbc1ffff640 1 --2- 172.21.15.105:0/3917040209 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc180ce390 0x7fbc18012dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.993 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.991+0000 7fbc1ffff640 1 -- 172.21.15.105:0/3917040209 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbc2006b630 con 0x7fbc180cdbd0 2024-09-21T01:44:37.993 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.992+0000 7fbc1ffff640 1 --2- 172.21.15.105:0/3917040209 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbc180cdbd0 0x7fbc18012890 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fbc1400eaf0 tx=0x7fbc140092a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:37.993 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.993+0000 7f98eeffd640 1 -- 172.21.15.105:0/1459717353 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f98f80b0040 con 0x7f98f000d8b0 2024-09-21T01:44:37.994 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.993+0000 7fbc1cff9640 1 -- 172.21.15.105:0/3917040209 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc14004020 con 0x7fbc180cdbd0 2024-09-21T01:44:37.994 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.993+0000 7fbc2533a640 1 -- 172.21.15.105:0/3917040209 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbc180d37c0 con 0x7fbc180cdbd0 2024-09-21T01:44:37.994 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.993+0000 7fbc2533a640 1 -- 172.21.15.105:0/3917040209 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbc180d4070 con 0x7fbc180cdbd0 2024-09-21T01:44:37.994 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.994+0000 7fbc1cff9640 1 -- 172.21.15.105:0/3917040209 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fbc14004270 con 0x7fbc180cdbd0 2024-09-21T01:44:37.995 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.994+0000 7fbc1cff9640 1 -- 172.21.15.105:0/3917040209 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbc14005020 con 0x7fbc180cdbd0 2024-09-21T01:44:37.995 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.994+0000 7fbc1cff9640 1 -- 172.21.15.105:0/3917040209 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7fbc1400a040 con 0x7fbc180cdbd0 2024-09-21T01:44:37.995 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.994+0000 7fbc1cff9640 1 -- 172.21.15.105:0/3917040209 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7fbc14004e30 con 0x7fbc180cdbd0 2024-09-21T01:44:37.995 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.993+0000 7f73a27fc640 1 -- 172.21.15.105:0/3498580097 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f73a4038070 con 0x7f73ac094090 2024-09-21T01:44:37.995 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.993+0000 7f98ecff9640 1 -- 172.21.15.105:0/1459717353 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f98e0005250 con 0x7f98f000d8b0 2024-09-21T01:44:37.995 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.994+0000 7f98eeffd640 1 -- 172.21.15.105:0/1459717353 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f98f80b5950 con 0x7f98f000d8b0 2024-09-21T01:44:37.995 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.994+0000 7f98eeffd640 1 -- 172.21.15.105:0/1459717353 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f98f80ab420 con 0x7f98f000d8b0 2024-09-21T01:44:37.995 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.994+0000 7f98eeffd640 1 -- 172.21.15.105:0/1459717353 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f98f80be780 con 0x7f98f000d8b0 2024-09-21T01:44:37.997 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.997+0000 7fbc2533a640 1 -- 172.21.15.105:0/3917040209 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbc00005250 con 0x7fbc180cdbd0 2024-09-21T01:44:37.997 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.997+0000 7f0b0a7fc640 1 -- 172.21.15.105:0/672714383 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0b241292f0 msgr2=0x7f0b24131b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:37.998 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.997+0000 7f0b0a7fc640 1 --2- 172.21.15.105:0/672714383 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0b241292f0 0x7f0b24131b20 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f0b1c00cc60 tx=0x7f0b1c00a430 comp rx=0 tx=0).stop 2024-09-21T01:44:37.998 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.997+0000 7f0b0a7fc640 1 -- 172.21.15.105:0/672714383 shutdown_connections 2024-09-21T01:44:37.998 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.997+0000 7f0b0a7fc640 1 --2- 172.21.15.105:0/672714383 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0b2412d9a0 0x7f0b24132060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.998 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.997+0000 7f0b0a7fc640 1 --2- 172.21.15.105:0/672714383 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0b241292f0 0x7f0b24131b20 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.998 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.997+0000 7f0b0a7fc640 1 --2- 172.21.15.105:0/672714383 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0b241289d0 0x7f0b241315e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:37.998 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.997+0000 7f0b0a7fc640 1 -- 172.21.15.105:0/672714383 >> 172.21.15.105:0/672714383 conn(0x7f0b2409aa30 msgr2=0x7f0b2406e800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:37.998 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.997+0000 7f0b0a7fc640 1 -- 172.21.15.105:0/672714383 shutdown_connections 2024-09-21T01:44:37.998 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.997+0000 7f0b0a7fc640 1 -- 172.21.15.105:0/672714383 wait complete. 2024-09-21T01:44:37.998 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:37.998+0000 7f98eeffd640 1 -- 172.21.15.105:0/1459717353 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f98f80b9020 con 0x7f98f000d8b0 2024-09-21T01:44:38.004 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.004+0000 7fbc1cff9640 1 -- 172.21.15.105:0/3917040209 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7fbc1401b020 con 0x7fbc180cdbd0 2024-09-21T01:44:38.006 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.005+0000 7f140a908640 1 -- 172.21.15.105:0/981140175 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f140408cc10 msgr2=0x7f140406ccb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.006 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.005+0000 7f140a908640 1 --2- 172.21.15.105:0/981140175 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f140408cc10 0x7f140406ccb0 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f13fc005fd0 tx=0x7f13fc02e600 comp rx=0 tx=0).stop 2024-09-21T01:44:38.006 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.005+0000 7f140a908640 1 -- 172.21.15.105:0/981140175 shutdown_connections 2024-09-21T01:44:38.006 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.005+0000 7f140a908640 1 --2- 172.21.15.105:0/981140175 >> v2:172.21.15.105:3300/0 conn(0x7f140408c290 0x7f1404096dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.006 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.005+0000 7f140a908640 1 --2- 172.21.15.105:0/981140175 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f140408cc10 0x7f140406ccb0 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.006 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.005+0000 7f140a908640 1 -- 172.21.15.105:0/981140175 >> 172.21.15.105:0/981140175 conn(0x7f140409a7f0 msgr2=0x7f140409ac00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:38.006 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.006+0000 7f140a908640 1 -- 172.21.15.105:0/981140175 shutdown_connections 2024-09-21T01:44:38.007 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.006+0000 7f140a908640 1 -- 172.21.15.105:0/981140175 wait complete. 2024-09-21T01:44:38.007 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.007+0000 7f140a908640 1 Processor -- start 2024-09-21T01:44:38.008 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.007+0000 7f140a908640 1 -- start start 2024-09-21T01:44:38.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.008+0000 7f140a908640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1404088a80 0x7f140407bfd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:38.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.008+0000 7f140a908640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f140408c290 0x7f140407c510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:38.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.008+0000 7f140a908640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f140408cc10 0x7f140407ca50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:38.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.008+0000 7f140a908640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1404081df0 con 0x7f1404088a80 2024-09-21T01:44:38.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.008+0000 7f140a908640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f14040817c0 con 0x7f140408cc10 2024-09-21T01:44:38.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.008+0000 7f140a908640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f1404096820 con 0x7f140408c290 2024-09-21T01:44:38.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.008+0000 7f1403fff640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1404088a80 0x7f140407bfd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:38.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.008+0000 7f1403fff640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1404088a80 0x7f140407bfd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:55482/0 (socket says 172.21.15.105:55482) 2024-09-21T01:44:38.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.008+0000 7f1403fff640 1 -- 172.21.15.105:0/1059381737 learned_addr learned my addr 172.21.15.105:0/1059381737 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:38.010 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'mds_debug_frag', 'True'], check_status=True, hostname='smithi105') 2024-09-21T01:44:38.010 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'mds_debug_scatterstat', 'True'], check_status=True, hostname='smithi105') 2024-09-21T01:44:38.010 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.008+0000 7f14037fe640 1 --2- 172.21.15.105:0/1059381737 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f140408c290 0x7f140407c510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:38.010 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.009+0000 7f1403fff640 1 -- 172.21.15.105:0/1059381737 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f140408c290 msgr2=0x7f140407c510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.011 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.009+0000 7f1403fff640 1 --2- 172.21.15.105:0/1059381737 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f140408c290 0x7f140407c510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.011 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.009+0000 7f1403fff640 1 -- 172.21.15.105:0/1059381737 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f140408cc10 msgr2=0x7f140407ca50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.016 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.009+0000 7f1408ea5640 1 --2- 172.21.15.105:0/1059381737 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f140408cc10 0x7f140407ca50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:38.016 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.016+0000 7f1403fff640 1 --2- 172.21.15.105:0/1059381737 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f140408cc10 0x7f140407ca50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.016 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.016+0000 7f1403fff640 1 -- 172.21.15.105:0/1059381737 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f13fc008830 con 0x7f1404088a80 2024-09-21T01:44:38.017 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.016+0000 7f1403fff640 1 --2- 172.21.15.105:0/1059381737 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1404088a80 0x7f140407bfd0 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f13f400c750 tx=0x7f13f400cc10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:38.017 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.016+0000 7f1408ea5640 1 --2- 172.21.15.105:0/1059381737 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f140408cc10 0x7f140407ca50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T01:44:38.017 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.017+0000 7f1400ff9640 1 -- 172.21.15.105:0/1059381737 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f13f4005270 con 0x7f1404088a80 2024-09-21T01:44:38.019 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.017+0000 7f140a908640 1 -- 172.21.15.105:0/1059381737 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f140407d060 con 0x7f1404088a80 2024-09-21T01:44:38.019 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.017+0000 7f140a908640 1 -- 172.21.15.105:0/1059381737 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f14041d3b50 con 0x7f1404088a80 2024-09-21T01:44:38.019 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.018+0000 7f1400ff9640 1 -- 172.21.15.105:0/1059381737 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f13f40054c0 con 0x7f1404088a80 2024-09-21T01:44:38.019 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.018+0000 7f1400ff9640 1 -- 172.21.15.105:0/1059381737 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f13f400a8a0 con 0x7f1404088a80 2024-09-21T01:44:38.019 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.019+0000 7f1400ff9640 1 -- 172.21.15.105:0/1059381737 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f13f40035f0 con 0x7f1404088a80 2024-09-21T01:44:38.019 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.019+0000 7f1400ff9640 1 -- 172.21.15.105:0/1059381737 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f13f401c940 con 0x7f1404088a80 2024-09-21T01:44:38.020 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.019+0000 7f13ea7fc640 1 -- 172.21.15.105:0/1059381737 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f13f0005250 con 0x7f1404088a80 2024-09-21T01:44:38.021 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.021+0000 7f14037fe640 1 --2- 172.21.15.105:0/1059381737 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f140408c290 0x7f140407c510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T01:44:38.023 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.023+0000 7f1400ff9640 1 -- 172.21.15.105:0/1059381737 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f13f4017070 con 0x7f1404088a80 2024-09-21T01:44:38.031 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'mds_op_complaint_time', '180'], check_status=True, hostname='smithi105') 2024-09-21T01:44:38.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.051+0000 7f18fdd2f640 1 -- 172.21.15.105:0/1796095297 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command([{prefix=config set, name=mds_op_complaint_time}] v 0) v1 -- 0x7f18d80052f0 con 0x7f18f806e670 2024-09-21T01:44:38.061 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.061+0000 7f18f57fa640 1 -- 172.21.15.105:0/1796095297 <== mon.2 v2:172.21.15.105:3302/0 8 ==== mon_command_ack([{prefix=config set, name=mds_op_complaint_time}]=0 v12)=0 v12) v1 ==== 121+0+0 (secure 0 0 0) 0x7f18ec021e40 con 0x7f18f806e670 2024-09-21T01:44:38.066 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.066+0000 7f18deffd640 1 -- 172.21.15.105:0/1796095297 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f18f806e670 msgr2=0x7f18f806eb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.066 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.066+0000 7f18deffd640 1 --2- 172.21.15.105:0/1796095297 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f18f806e670 0x7f18f806eb00 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f18ec00ecb0 tx=0x7f18ec00c6a0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.067 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.066+0000 7f18deffd640 1 -- 172.21.15.105:0/1796095297 shutdown_connections 2024-09-21T01:44:38.067 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.066+0000 7f18deffd640 1 --2- 172.21.15.105:0/1796095297 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f18f806e670 0x7f18f806eb00 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.067 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.066+0000 7f18deffd640 1 --2- 172.21.15.105:0/1796095297 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f18f8135ea0 0x7f18f806e130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.067 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.066+0000 7f18deffd640 1 --2- 172.21.15.105:0/1796095297 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f18f812d100 0x7f18f80a7e30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.067 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.066+0000 7f18deffd640 1 -- 172.21.15.105:0/1796095297 >> 172.21.15.105:0/1796095297 conn(0x7f18f806bc90 msgr2=0x7f18f8099310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:38.067 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.066+0000 7f18deffd640 1 -- 172.21.15.105:0/1796095297 shutdown_connections 2024-09-21T01:44:38.067 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.066+0000 7f18deffd640 1 -- 172.21.15.105:0/1796095297 wait complete. 2024-09-21T01:44:38.083 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'mds_verify_scatter', 'True'], check_status=True, hostname='smithi105') 2024-09-21T01:44:38.125 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.125+0000 7f026a067640 1 -- 172.21.15.105:0/2199543376 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command([{prefix=config set, name=mds_verify_scatter}] v 0) v1 -- 0x7f0244005870 con 0x7f025c018fb0 2024-09-21T01:44:38.126 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.123+0000 7fadd9c2d640 1 -- 172.21.15.105:0/677805864 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command([{prefix=config set, name=rados_osd_op_timeout}] v 0) v1 -- 0x7fadb40050c0 con 0x7fadcc0afdd0 2024-09-21T01:44:38.137 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.137+0000 7f0259ffb640 1 -- 172.21.15.105:0/2199543376 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{prefix=config set, name=mds_verify_scatter}]=0 v13)=0 v13) v1 ==== 119+0+0 (secure 0 0 0) 0x7f02640b3dd0 con 0x7f025c018fb0 2024-09-21T01:44:38.145 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.145+0000 7f026a067640 1 -- 172.21.15.105:0/2199543376 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f025c018fb0 msgr2=0x7f025c0193f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.146 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.145+0000 7f026a067640 1 --2- 172.21.15.105:0/2199543376 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f025c018fb0 0x7f025c0193f0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f0264069bd0 tx=0x7f02640a9940 comp rx=0 tx=0).stop 2024-09-21T01:44:38.146 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.146+0000 7fadd17fa640 1 -- 172.21.15.105:0/677805864 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{prefix=config set, name=rados_osd_op_timeout}]=0 v14)=0 v14) v1 ==== 120+0+0 (secure 0 0 0) 0x7fadd4073630 con 0x7fadcc0afdd0 2024-09-21T01:44:38.147 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.146+0000 7f026a067640 1 -- 172.21.15.105:0/2199543376 shutdown_connections 2024-09-21T01:44:38.147 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.146+0000 7f026a067640 1 --2- 172.21.15.105:0/2199543376 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f025c151900 0x7f025c153cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.147 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.146+0000 7f026a067640 1 --2- 172.21.15.105:0/2199543376 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f025c018fb0 0x7f025c0193f0 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.147 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.146+0000 7f026a067640 1 --2- 172.21.15.105:0/2199543376 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f025c0b34b0 0x7f025c018a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.147 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.146+0000 7f026a067640 1 -- 172.21.15.105:0/2199543376 >> 172.21.15.105:0/2199543376 conn(0x7f025c005af0 msgr2=0x7f025c0bd0a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:38.148 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.146+0000 7f026a067640 1 -- 172.21.15.105:0/2199543376 shutdown_connections 2024-09-21T01:44:38.148 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.147+0000 7f026a067640 1 -- 172.21.15.105:0/2199543376 wait complete. 2024-09-21T01:44:38.152 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.152+0000 7fadd9c2d640 1 -- 172.21.15.105:0/677805864 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fadcc0afdd0 msgr2=0x7fadcc0b1490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.153 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.152+0000 7fadd9c2d640 1 --2- 172.21.15.105:0/677805864 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fadcc0afdd0 0x7fadcc0b1490 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7fadd40639a0 tx=0x7fadd40667b0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.153 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.152+0000 7fadd9c2d640 1 -- 172.21.15.105:0/677805864 shutdown_connections 2024-09-21T01:44:38.153 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.152+0000 7fadd9c2d640 1 --2- 172.21.15.105:0/677805864 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fadcc0afdd0 0x7fadcc0b1490 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.153 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.152+0000 7fadd9c2d640 1 --2- 172.21.15.105:0/677805864 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fadcc00e3b0 0x7fadcc0b0f50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.153 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.152+0000 7fadd9c2d640 1 --2- 172.21.15.105:0/677805864 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fadcc00da90 0x7fadcc0b0a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.153 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.152+0000 7fadd9c2d640 1 -- 172.21.15.105:0/677805864 >> 172.21.15.105:0/677805864 conn(0x7fadcc005af0 msgr2=0x7fadcc00a5b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:38.153 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.152+0000 7fadd9c2d640 1 -- 172.21.15.105:0/677805864 shutdown_connections 2024-09-21T01:44:38.153 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.152+0000 7fadd9c2d640 1 -- 172.21.15.105:0/677805864 wait complete. 2024-09-21T01:44:38.154 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.135+0000 7feeb3791640 1 -- 172.21.15.105:0/270538509 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command([{prefix=config set, name=rados_mon_op_timeout}] v 0) v1 -- 0x7fee8c005870 con 0x7feea40cdb80 2024-09-21T01:44:38.156 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.155+0000 7f92560f8640 1 -- 172.21.15.105:0/2149734158 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command([{prefix=config set, name=debug_client}] v 0) v1 -- 0x7f9250067260 con 0x7f925008c200 2024-09-21T01:44:38.162 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.162+0000 7feea1ffb640 1 -- 172.21.15.105:0/270538509 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{prefix=config set, name=rados_mon_op_timeout}]=0 v15)=0 v15) v1 ==== 120+0+0 (secure 0 0 0) 0x7feea8004960 con 0x7feea40cdb80 2024-09-21T01:44:38.168 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.168+0000 7f9245ffb640 1 -- 172.21.15.105:0/2149734158 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{prefix=config set, name=debug_client}]=0 v16)=0 v16) v1 ==== 111+0+0 (secure 0 0 0) 0x7f924c005460 con 0x7f925008c200 2024-09-21T01:44:38.173 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.171+0000 7feeb3791640 1 -- 172.21.15.105:0/270538509 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feea40cdb80 msgr2=0x7feea4012530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.173 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.171+0000 7feeb3791640 1 --2- 172.21.15.105:0/270538509 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feea40cdb80 0x7feea4012530 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7feea800fea0 tx=0x7feea800ff90 comp rx=0 tx=0).stop 2024-09-21T01:44:38.173 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.171+0000 7feeb3791640 1 -- 172.21.15.105:0/270538509 shutdown_connections 2024-09-21T01:44:38.174 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.171+0000 7feeb3791640 1 --2- 172.21.15.105:0/270538509 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7feea4012fb0 0x7feea4013440 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.174 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.171+0000 7feeb3791640 1 --2- 172.21.15.105:0/270538509 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feea40ce340 0x7feea4012a70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.174 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.171+0000 7feeb3791640 1 --2- 172.21.15.105:0/270538509 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feea40cdb80 0x7feea4012530 secure :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7feea800fea0 tx=0x7feea800ff90 comp rx=0 tx=0).stop 2024-09-21T01:44:38.174 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.171+0000 7feeb3791640 1 -- 172.21.15.105:0/270538509 >> 172.21.15.105:0/270538509 conn(0x7feea4005aa0 msgr2=0x7feea40cf640 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:38.174 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.173+0000 7feeb3791640 1 -- 172.21.15.105:0/270538509 shutdown_connections 2024-09-21T01:44:38.174 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.173+0000 7feeb3791640 1 -- 172.21.15.105:0/270538509 wait complete. 2024-09-21T01:44:38.174 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.173+0000 7f1c65391640 1 -- 172.21.15.105:0/4207177331 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command([{prefix=config set, name=debug_ms}] v 0) v1 -- 0x7f1c400050c0 con 0x7f1c6006c9a0 2024-09-21T01:44:38.180 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.180+0000 7f98ecff9640 1 -- 172.21.15.105:0/1459717353 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command([{prefix=config set, name=osd_op_complaint_time}] v 0) v1 -- 0x7f98e00057d0 con 0x7f98f000d8b0 2024-09-21T01:44:38.182 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.182+0000 7f92560f8640 1 -- 172.21.15.105:0/2149734158 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f925008c200 msgr2=0x7f92501384b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.182 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.182+0000 7f92560f8640 1 --2- 172.21.15.105:0/2149734158 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f925008c200 0x7f92501384b0 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f924c00e9c0 tx=0x7f924c00ee80 comp rx=0 tx=0).stop 2024-09-21T01:44:38.183 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.182+0000 7f92560f8640 1 -- 172.21.15.105:0/2149734158 shutdown_connections 2024-09-21T01:44:38.183 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.182+0000 7f92560f8640 1 --2- 172.21.15.105:0/2149734158 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f92501295e0 0x7f9250125c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.183 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.182+0000 7f92560f8640 1 --2- 172.21.15.105:0/2149734158 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f9250093080 0x7f9250125730 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.183 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.182+0000 7f92560f8640 1 --2- 172.21.15.105:0/2149734158 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f925008c200 0x7f92501384b0 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.183 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.182+0000 7f92560f8640 1 -- 172.21.15.105:0/2149734158 >> 172.21.15.105:0/2149734158 conn(0x7f925009a7f0 msgr2=0x7f925012a500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:38.184 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.184+0000 7fbc2533a640 1 -- 172.21.15.105:0/3917040209 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command([{prefix=config set, name=osd_op_complaint_time}] v 0) v1 -- 0x7fbc000050c0 con 0x7fbc180cdbd0 2024-09-21T01:44:38.185 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'osd_op_complaint_time', '180'], check_status=True, hostname='smithi105') 2024-09-21T01:44:38.185 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.184+0000 7f92560f8640 1 -- 172.21.15.105:0/2149734158 shutdown_connections 2024-09-21T01:44:38.185 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.185+0000 7f92560f8640 1 -- 172.21.15.105:0/2149734158 wait complete. 2024-09-21T01:44:38.186 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.186+0000 7f1c477fe640 1 -- 172.21.15.105:0/4207177331 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{prefix=config set, name=debug_ms}]=0 v17)=0 v17) v1 ==== 106+0+0 (secure 0 0 0) 0x7f1c58004bf0 con 0x7f1c6006c9a0 2024-09-21T01:44:38.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.188+0000 7f1c65391640 1 -- 172.21.15.105:0/4207177331 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1c6006c9a0 msgr2=0x7f1c6008e050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.188+0000 7f1c65391640 1 --2- 172.21.15.105:0/4207177331 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1c6006c9a0 0x7f1c6008e050 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f1c5800cd10 tx=0x7f1c5800a620 comp rx=0 tx=0).stop 2024-09-21T01:44:38.189 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.188+0000 7f1c65391640 1 -- 172.21.15.105:0/4207177331 shutdown_connections 2024-09-21T01:44:38.189 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.188+0000 7f1c65391640 1 --2- 172.21.15.105:0/4207177331 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f1c60128be0 0x7f1c6008ead0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.189 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.188+0000 7f1c65391640 1 --2- 172.21.15.105:0/4207177331 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1c601282c0 0x7f1c6008e590 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.189 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.188+0000 7f1c65391640 1 --2- 172.21.15.105:0/4207177331 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1c6006c9a0 0x7f1c6008e050 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.189 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.188+0000 7f1c65391640 1 -- 172.21.15.105:0/4207177331 >> 172.21.15.105:0/4207177331 conn(0x7f1c60099470 msgr2=0x7f1c60089000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:38.189 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.188+0000 7f1c65391640 1 -- 172.21.15.105:0/4207177331 shutdown_connections 2024-09-21T01:44:38.189 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.188+0000 7f1c65391640 1 -- 172.21.15.105:0/4207177331 wait complete. 2024-09-21T01:44:38.190 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.176+0000 7f73b2a17640 1 -- 172.21.15.105:0/3498580097 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command([{prefix=config set, name=mon_op_complaint_time}] v 0) v1 -- 0x7f73940050c0 con 0x7f73ac094090 2024-09-21T01:44:38.192 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.192+0000 7f33e120a640 1 -- 172.21.15.105:0/3661563747 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command([{prefix=config set, name=debug_mgr}] v 0) v1 -- 0x7f33bc0050c0 con 0x7f33d40a9e00 2024-09-21T01:44:38.196 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.196+0000 7fbc1cff9640 1 -- 172.21.15.105:0/3917040209 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{prefix=config set, name=osd_op_complaint_time}]=0 v18)=0 v18) v1 ==== 121+0+0 (secure 0 0 0) 0x7fbc14005610 con 0x7fbc180cdbd0 2024-09-21T01:44:38.198 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.197+0000 7fbc2533a640 1 -- 172.21.15.105:0/3917040209 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbc180cdbd0 msgr2=0x7fbc18012890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.198 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.197+0000 7fbc2533a640 1 --2- 172.21.15.105:0/3917040209 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbc180cdbd0 0x7fbc18012890 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fbc1400eaf0 tx=0x7fbc140092a0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.198 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.197+0000 7fbc2533a640 1 -- 172.21.15.105:0/3917040209 shutdown_connections 2024-09-21T01:44:38.198 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.197+0000 7fbc2533a640 1 --2- 172.21.15.105:0/3917040209 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbc18013310 0x7fbc180e4010 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.198 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.197+0000 7fbc2533a640 1 --2- 172.21.15.105:0/3917040209 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbc180ce390 0x7fbc18012dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.198 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.197+0000 7fbc2533a640 1 --2- 172.21.15.105:0/3917040209 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbc180cdbd0 0x7fbc18012890 secure :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fbc1400eaf0 tx=0x7fbc140092a0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.198 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.197+0000 7fbc2533a640 1 -- 172.21.15.105:0/3917040209 >> 172.21.15.105:0/3917040209 conn(0x7fbc18005af0 msgr2=0x7fbc180ca220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:38.199 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.199+0000 7fbc2533a640 1 -- 172.21.15.105:0/3917040209 shutdown_connections 2024-09-21T01:44:38.200 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.199+0000 7fbc2533a640 1 -- 172.21.15.105:0/3917040209 wait complete. 2024-09-21T01:44:38.205 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.204+0000 7f98eeffd640 1 -- 172.21.15.105:0/1459717353 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{prefix=config set, name=osd_op_complaint_time}]=0 v19)=0 v19) v1 ==== 121+0+0 (secure 0 0 0) 0x7f98f80be940 con 0x7f98f000d8b0 2024-09-21T01:44:38.205 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.204+0000 7f24fb6e5640 1 -- 172.21.15.105:0/1798968300 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command([{prefix=config set, name=client_mount_timeout}] v 0) v1 -- 0x7f24d40050c0 con 0x7f24f4094f50 2024-09-21T01:44:38.206 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.205+0000 7f98ecff9640 1 -- 172.21.15.105:0/1459717353 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f98f000d8b0 msgr2=0x7f98f0142190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.206 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.205+0000 7f98ecff9640 1 --2- 172.21.15.105:0/1459717353 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f98f000d8b0 0x7f98f0142190 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f98f8068e50 tx=0x7f98f80aad80 comp rx=0 tx=0).stop 2024-09-21T01:44:38.206 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.205+0000 7f98ecff9640 1 -- 172.21.15.105:0/1459717353 shutdown_connections 2024-09-21T01:44:38.206 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.205+0000 7f98ecff9640 1 --2- 172.21.15.105:0/1459717353 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f98f000e650 0x7f98f0142c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.206 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.205+0000 7f98ecff9640 1 --2- 172.21.15.105:0/1459717353 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f98f000e1d0 0x7f98f01426d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.206 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.205+0000 7f98ecff9640 1 --2- 172.21.15.105:0/1459717353 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f98f000d8b0 0x7f98f0142190 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.206 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.205+0000 7f98ecff9640 1 -- 172.21.15.105:0/1459717353 >> 172.21.15.105:0/1459717353 conn(0x7f98f0006710 msgr2=0x7f98f00123b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:38.206 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.206+0000 7f98ecff9640 1 -- 172.21.15.105:0/1459717353 shutdown_connections 2024-09-21T01:44:38.206 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.206+0000 7f98ecff9640 1 -- 172.21.15.105:0/1459717353 wait complete. 2024-09-21T01:44:38.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.211+0000 7f33d8ff9640 1 -- 172.21.15.105:0/3661563747 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{prefix=config set, name=debug_mgr}]=0 v20)=0 v20) v1 ==== 108+0+0 (secure 0 0 0) 0x7f33c8004c10 con 0x7f33d40a9e00 2024-09-21T01:44:38.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.212+0000 7f33e120a640 1 -- 172.21.15.105:0/3661563747 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f33d40a9e00 msgr2=0x7f33d40aa240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.212+0000 7f33e120a640 1 --2- 172.21.15.105:0/3661563747 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f33d40a9e00 0x7f33d40aa240 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f33c800ca30 tx=0x7f33c800cef0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.212+0000 7f33e120a640 1 -- 172.21.15.105:0/3661563747 shutdown_connections 2024-09-21T01:44:38.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.212+0000 7f33e120a640 1 --2- 172.21.15.105:0/3661563747 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f33d40aa780 0x7f33d40aac00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.212+0000 7f33e120a640 1 --2- 172.21.15.105:0/3661563747 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f33d40a9e00 0x7f33d40aa240 secure :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f33c800ca30 tx=0x7f33c800cef0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.212+0000 7f33e120a640 1 --2- 172.21.15.105:0/3661563747 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f33d40b1d60 0x7f33d40beba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.212+0000 7f33e120a640 1 -- 172.21.15.105:0/3661563747 >> 172.21.15.105:0/3661563747 conn(0x7f33d4005af0 msgr2=0x7f33d401d090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:38.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.213+0000 7f33e120a640 1 -- 172.21.15.105:0/3661563747 shutdown_connections 2024-09-21T01:44:38.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.213+0000 7f33e120a640 1 -- 172.21.15.105:0/3661563747 wait complete. 2024-09-21T01:44:38.218 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.218+0000 7f73a27fc640 1 -- 172.21.15.105:0/3498580097 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{prefix=config set, name=mon_op_complaint_time}]=0 v21)=0 v21) v1 ==== 121+0+0 (secure 0 0 0) 0x7f73a4004520 con 0x7f73ac094090 2024-09-21T01:44:38.219 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.219+0000 7f73b2a17640 1 -- 172.21.15.105:0/3498580097 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f73ac094090 msgr2=0x7f73ac1d0c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.220 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.219+0000 7f73b2a17640 1 --2- 172.21.15.105:0/3498580097 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f73ac094090 0x7f73ac1d0c40 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f73a4009e50 tx=0x7f73a402cba0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.220 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.220+0000 7f73b2a17640 1 -- 172.21.15.105:0/3498580097 shutdown_connections 2024-09-21T01:44:38.220 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.220+0000 7f73b2a17640 1 --2- 172.21.15.105:0/3498580097 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f73ac08f320 0x7f73ac08f7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.220 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.220+0000 7f73b2a17640 1 --2- 172.21.15.105:0/3498580097 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f73ac094090 0x7f73ac1d0c40 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.220 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.220+0000 7f73b2a17640 1 --2- 172.21.15.105:0/3498580097 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f73ac088910 0x7f73ac1d0700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.221 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.220+0000 7f73b2a17640 1 -- 172.21.15.105:0/3498580097 >> 172.21.15.105:0/3498580097 conn(0x7f73ac09a610 msgr2=0x7f73ac087710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:38.221 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.221+0000 7f73b2a17640 1 -- 172.21.15.105:0/3498580097 shutdown_connections 2024-09-21T01:44:38.222 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.221+0000 7f73b2a17640 1 -- 172.21.15.105:0/3498580097 wait complete. 2024-09-21T01:44:38.224 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'rados_mon_op_timeout', '900'], check_status=True, hostname='smithi105') 2024-09-21T01:44:38.224 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mds', 'rados_osd_op_timeout', '900'], check_status=True, hostname='smithi105') 2024-09-21T01:44:38.224 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mgr', 'client_mount_timeout', '30'], check_status=True, hostname='smithi105') 2024-09-21T01:44:38.224 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.224+0000 7f24e9ffb640 1 -- 172.21.15.105:0/1798968300 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{prefix=config set, name=client_mount_timeout}]=0 v22)=0 v22) v1 ==== 119+0+0 (secure 0 0 0) 0x7f24f0028030 con 0x7f24f4094f50 2024-09-21T01:44:38.226 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.225+0000 7f24fb6e5640 1 -- 172.21.15.105:0/1798968300 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f24f4094f50 msgr2=0x7f24f4093f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.226 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.225+0000 7f24fb6e5640 1 --2- 172.21.15.105:0/1798968300 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f24f4094f50 0x7f24f4093f60 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f24f000b450 tx=0x7f24f000b910 comp rx=0 tx=0).stop 2024-09-21T01:44:38.226 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.225+0000 7f24fb6e5640 1 -- 172.21.15.105:0/1798968300 shutdown_connections 2024-09-21T01:44:38.226 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.225+0000 7f24fb6e5640 1 --2- 172.21.15.105:0/1798968300 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f24f40949e0 0x7f24f4079a60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.226 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.225+0000 7f24fb6e5640 1 --2- 172.21.15.105:0/1798968300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f24f409ad20 0x7f24f40944a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.226 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.225+0000 7f24fb6e5640 1 --2- 172.21.15.105:0/1798968300 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f24f4094f50 0x7f24f4093f60 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.226 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.225+0000 7f24fb6e5640 1 -- 172.21.15.105:0/1798968300 >> 172.21.15.105:0/1798968300 conn(0x7f24f4098b10 msgr2=0x7f24f408aff0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:38.226 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.225+0000 7f24fb6e5640 1 -- 172.21.15.105:0/1798968300 shutdown_connections 2024-09-21T01:44:38.226 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.225+0000 7f24fb6e5640 1 -- 172.21.15.105:0/1798968300 wait complete. 2024-09-21T01:44:38.237 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:38.237+0000 7fc75c220040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T01:44:38.238 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:38.238+0000 7f43c91dd040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T01:44:38.242 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mgr', 'debug_client', '20'], check_status=True, hostname='smithi105') 2024-09-21T01:44:38.242 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mgr', 'debug_mgr', '20'], check_status=True, hostname='smithi105') 2024-09-21T01:44:38.242 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mgr', 'debug_ms', '1'], check_status=True, hostname='smithi105') 2024-09-21T01:44:38.242 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mgr', 'mon_warn_on_pool_no_app', 'False'], check_status=True, hostname='smithi105') 2024-09-21T01:44:38.244 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.244+0000 7f13ea7fc640 1 -- 172.21.15.105:0/1059381737 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command([{prefix=config set, name=mon_warn_on_pool_no_app}] v 0) v1 -- 0x7f13f00057d0 con 0x7f1404088a80 2024-09-21T01:44:38.248 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.247+0000 7f1400ff9640 1 -- 172.21.15.105:0/1059381737 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_warn_on_pool_no_app}]=0 v23)=0 v23) v1 ==== 125+0+0 (secure 0 0 0) 0x7f13f401cb50 con 0x7f1404088a80 2024-09-21T01:44:38.249 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.248+0000 7f13ea7fc640 1 -- 172.21.15.105:0/1059381737 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1404088a80 msgr2=0x7f140407bfd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.249 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.248+0000 7f13ea7fc640 1 --2- 172.21.15.105:0/1059381737 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1404088a80 0x7f140407bfd0 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f13f400c750 tx=0x7f13f400cc10 comp rx=0 tx=0).stop 2024-09-21T01:44:38.249 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.248+0000 7f13ea7fc640 1 -- 172.21.15.105:0/1059381737 shutdown_connections 2024-09-21T01:44:38.249 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.248+0000 7f13ea7fc640 1 --2- 172.21.15.105:0/1059381737 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f140408cc10 0x7f140407ca50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.249 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.248+0000 7f13ea7fc640 1 --2- 172.21.15.105:0/1059381737 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f140408c290 0x7f140407c510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.249 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.248+0000 7f13ea7fc640 1 --2- 172.21.15.105:0/1059381737 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1404088a80 0x7f140407bfd0 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.249 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.248+0000 7f13ea7fc640 1 -- 172.21.15.105:0/1059381737 >> 172.21.15.105:0/1059381737 conn(0x7f140409a7f0 msgr2=0x7f140408a2a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:38.249 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.248+0000 7f13ea7fc640 1 -- 172.21.15.105:0/1059381737 shutdown_connections 2024-09-21T01:44:38.250 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.249+0000 7f13ea7fc640 1 -- 172.21.15.105:0/1059381737 wait complete. 2024-09-21T01:44:38.260 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'mon', 'mon_op_complaint_time', '120'], check_status=True, hostname='smithi105') 2024-09-21T01:44:38.260 DEBUG:tasks.ceph:waiting for RemoteProcess(client=, args=['sudo', 'ceph', '--cluster', 'ceph', 'config', 'set', 'osd', 'osd_op_complaint_time', '180'], check_status=True, hostname='smithi105') 2024-09-21T01:44:38.260 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config dump 2024-09-21T01:44:38.354 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:38.354+0000 7fc75c220040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T01:44:38.355 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:38.354+0000 7f43c91dd040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T01:44:38.399 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.398+0000 7f3883ce9640 1 -- 172.21.15.105:0/3625720773 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f387c099b90 msgr2=0x7f387c1342a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.399 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.398+0000 7f3883ce9640 1 --2- 172.21.15.105:0/3625720773 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f387c099b90 0x7f387c1342a0 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f3874009ca0 tx=0x7f387402a950 comp rx=0 tx=0).stop 2024-09-21T01:44:38.399 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.398+0000 7f3883ce9640 1 -- 172.21.15.105:0/3625720773 shutdown_connections 2024-09-21T01:44:38.399 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.398+0000 7f3883ce9640 1 --2- 172.21.15.105:0/3625720773 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f387c099b90 0x7f387c1342a0 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.399 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.398+0000 7f3883ce9640 1 --2- 172.21.15.105:0/3625720773 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f387c128da0 0x7f387c129180 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.399 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.398+0000 7f3883ce9640 1 -- 172.21.15.105:0/3625720773 >> 172.21.15.105:0/3625720773 conn(0x7f387c09b9a0 msgr2=0x7f387c09bdb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:38.400 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.398+0000 7f3883ce9640 1 -- 172.21.15.105:0/3625720773 shutdown_connections 2024-09-21T01:44:38.400 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.399+0000 7f3883ce9640 1 -- 172.21.15.105:0/3625720773 wait complete. 2024-09-21T01:44:38.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.399+0000 7f3883ce9640 1 Processor -- start 2024-09-21T01:44:38.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.399+0000 7f3883ce9640 1 -- start start 2024-09-21T01:44:38.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.400+0000 7f3883ce9640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f387c099b90 0x7f387c12f830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:38.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.400+0000 7f3883ce9640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f387c128da0 0x7f387c12fd70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:38.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.400+0000 7f3883ce9640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f387c1296c0 0x7f387c1302b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:38.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.400+0000 7f3883ce9640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f387c132420 con 0x7f387c128da0 2024-09-21T01:44:38.403 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.400+0000 7f3883ce9640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f387c1312c0 con 0x7f387c099b90 2024-09-21T01:44:38.403 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.400+0000 7f3883ce9640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f387c133290 con 0x7f387c1296c0 2024-09-21T01:44:38.403 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.400+0000 7f3882286640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f387c1296c0 0x7f387c1302b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:38.403 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.400+0000 7f3881284640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f387c128da0 0x7f387c12fd70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:38.403 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.400+0000 7f3882286640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f387c1296c0 0x7f387c1302b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:45600/0 (socket says 172.21.15.105:45600) 2024-09-21T01:44:38.403 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.400+0000 7f3882286640 1 -- 172.21.15.105:0/3875404967 learned_addr learned my addr 172.21.15.105:0/3875404967 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:38.403 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.400+0000 7f3881a85640 1 --2- 172.21.15.105:0/3875404967 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f387c099b90 0x7f387c12f830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:38.403 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.400+0000 7f3881284640 1 -- 172.21.15.105:0/3875404967 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f387c1296c0 msgr2=0x7f387c1302b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.403 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.400+0000 7f3881284640 1 --2- 172.21.15.105:0/3875404967 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f387c1296c0 0x7f387c1302b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.403 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.400+0000 7f3881284640 1 -- 172.21.15.105:0/3875404967 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f387c099b90 msgr2=0x7f387c12f830 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.403 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.400+0000 7f3881284640 1 --2- 172.21.15.105:0/3875404967 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f387c099b90 0x7f387c12f830 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.403 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.400+0000 7f3881284640 1 -- 172.21.15.105:0/3875404967 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3878009490 con 0x7f387c128da0 2024-09-21T01:44:38.403 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.400+0000 7f3882286640 1 --2- 172.21.15.105:0/3875404967 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f387c1296c0 0x7f387c1302b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:38.403 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.400+0000 7f3881284640 1 --2- 172.21.15.105:0/3875404967 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f387c128da0 0x7f387c12fd70 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f387c129aa0 tx=0x7f386c00c890 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:38.404 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.401+0000 7f38727fc640 1 -- 172.21.15.105:0/3875404967 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f386c0035f0 con 0x7f387c128da0 2024-09-21T01:44:38.404 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.401+0000 7f38727fc640 1 -- 172.21.15.105:0/3875404967 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f386c003770 con 0x7f387c128da0 2024-09-21T01:44:38.404 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.401+0000 7f3883ce9640 1 -- 172.21.15.105:0/3875404967 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3874009900 con 0x7f387c128da0 2024-09-21T01:44:38.404 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.401+0000 7f38727fc640 1 -- 172.21.15.105:0/3875404967 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f386c003b80 con 0x7f387c128da0 2024-09-21T01:44:38.404 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.401+0000 7f3883ce9640 1 -- 172.21.15.105:0/3875404967 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f387c131cd0 con 0x7f387c128da0 2024-09-21T01:44:38.404 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.401+0000 7f38727fc640 1 -- 172.21.15.105:0/3875404967 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f386c01c440 con 0x7f387c128da0 2024-09-21T01:44:38.404 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.401+0000 7f38727fc640 1 -- 172.21.15.105:0/3875404967 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f386c012780 con 0x7f387c128da0 2024-09-21T01:44:38.404 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.402+0000 7f3883ce9640 1 -- 172.21.15.105:0/3875404967 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f387c067260 con 0x7f387c128da0 2024-09-21T01:44:38.404 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.404+0000 7f38727fc640 1 -- 172.21.15.105:0/3875404967 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f386c017070 con 0x7f387c128da0 2024-09-21T01:44:38.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.462+0000 7f3883ce9640 1 -- 172.21.15.105:0/3875404967 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "config dump"} v 0) v1 -- 0x7f387c12a1b0 con 0x7f387c128da0 2024-09-21T01:44:38.463 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.462+0000 7f38727fc640 1 -- 172.21.15.105:0/3875404967 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "config dump"}]=0 v23) v1 ==== 59+0+1472 (secure 0 0 0) 0x7f386c0129f0 con 0x7f387c128da0 2024-09-21T01:44:38.463 INFO:teuthology.orchestra.run.smithi105.stdout:WHO MASK LEVEL OPTION VALUE RO 2024-09-21T01:44:38.463 INFO:teuthology.orchestra.run.smithi105.stdout:mon advanced mon_op_complaint_time 120 2024-09-21T01:44:38.463 INFO:teuthology.orchestra.run.smithi105.stdout:mgr advanced client_mount_timeout 30 2024-09-21T01:44:38.463 INFO:teuthology.orchestra.run.smithi105.stdout:mgr advanced debug_client 20/20 2024-09-21T01:44:38.463 INFO:teuthology.orchestra.run.smithi105.stdout:mgr advanced debug_mgr 20/20 2024-09-21T01:44:38.463 INFO:teuthology.orchestra.run.smithi105.stdout:mgr advanced debug_ms 1/1 2024-09-21T01:44:38.463 INFO:teuthology.orchestra.run.smithi105.stdout:mgr dev mon_warn_on_pool_no_app false 2024-09-21T01:44:38.464 INFO:teuthology.orchestra.run.smithi105.stdout:osd advanced osd_op_complaint_time 180.000000 2024-09-21T01:44:38.464 INFO:teuthology.orchestra.run.smithi105.stdout:mds advanced debug_mds 20/20 2024-09-21T01:44:38.464 INFO:teuthology.orchestra.run.smithi105.stdout:mds advanced debug_mds_balancer 20/20 2024-09-21T01:44:38.464 INFO:teuthology.orchestra.run.smithi105.stdout:mds advanced debug_ms 2/2 2024-09-21T01:44:38.464 INFO:teuthology.orchestra.run.smithi105.stdout:mds dev mds_debug_frag true 2024-09-21T01:44:38.464 INFO:teuthology.orchestra.run.smithi105.stdout:mds dev mds_debug_scatterstat true 2024-09-21T01:44:38.464 INFO:teuthology.orchestra.run.smithi105.stdout:mds advanced mds_op_complaint_time 180.000000 2024-09-21T01:44:38.464 INFO:teuthology.orchestra.run.smithi105.stdout:mds dev mds_verify_scatter true 2024-09-21T01:44:38.464 INFO:teuthology.orchestra.run.smithi105.stdout:mds advanced osd_op_complaint_time 180.000000 2024-09-21T01:44:38.464 INFO:teuthology.orchestra.run.smithi105.stdout:mds advanced rados_mon_op_timeout 900 2024-09-21T01:44:38.464 INFO:teuthology.orchestra.run.smithi105.stdout:mds advanced rados_osd_op_timeout 900 2024-09-21T01:44:38.464 INFO:teuthology.orchestra.run.smithi105.stdout:client advanced client_mount_timeout 600 2024-09-21T01:44:38.464 INFO:teuthology.orchestra.run.smithi105.stdout:client advanced debug_client 20/20 2024-09-21T01:44:38.464 INFO:teuthology.orchestra.run.smithi105.stdout:client advanced debug_ms 1/1 2024-09-21T01:44:38.464 INFO:teuthology.orchestra.run.smithi105.stdout:client advanced rados_mon_op_timeout 900 2024-09-21T01:44:38.465 INFO:teuthology.orchestra.run.smithi105.stdout:client advanced rados_osd_op_timeout 900 2024-09-21T01:44:38.465 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.464+0000 7f3883ce9640 1 -- 172.21.15.105:0/3875404967 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f387c128da0 msgr2=0x7f387c12fd70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.465 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.464+0000 7f3883ce9640 1 --2- 172.21.15.105:0/3875404967 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f387c128da0 0x7f387c12fd70 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f387c129aa0 tx=0x7f386c00c890 comp rx=0 tx=0).stop 2024-09-21T01:44:38.466 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.464+0000 7f3883ce9640 1 -- 172.21.15.105:0/3875404967 shutdown_connections 2024-09-21T01:44:38.466 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.464+0000 7f3883ce9640 1 --2- 172.21.15.105:0/3875404967 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f387c1296c0 0x7f387c1302b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.466 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.464+0000 7f3883ce9640 1 --2- 172.21.15.105:0/3875404967 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f387c128da0 0x7f387c12fd70 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.466 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.464+0000 7f3883ce9640 1 --2- 172.21.15.105:0/3875404967 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f387c099b90 0x7f387c12f830 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.466 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.464+0000 7f3883ce9640 1 -- 172.21.15.105:0/3875404967 >> 172.21.15.105:0/3875404967 conn(0x7f387c09b9a0 msgr2=0x7f387c12c6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:38.466 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.464+0000 7f3883ce9640 1 -- 172.21.15.105:0/3875404967 shutdown_connections 2024-09-21T01:44:38.466 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.464+0000 7f3883ce9640 1 -- 172.21.15.105:0/3875404967 wait complete. 2024-09-21T01:44:38.477 INFO:tasks.ceph:Setting crush tunables to default 2024-09-21T01:44:38.478 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph osd crush tunables default 2024-09-21T01:44:38.578 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.577+0000 7f4fb677d640 1 -- 172.21.15.105:0/596955160 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4fb0131860 msgr2=0x7f4fb0133c50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.578 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.577+0000 7f4fb677d640 1 --2- 172.21.15.105:0/596955160 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4fb0131860 0x7f4fb0133c50 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f4fa8009be0 tx=0x7f4fa802a770 comp rx=0 tx=0).stop 2024-09-21T01:44:38.578 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.577+0000 7f4fb677d640 1 -- 172.21.15.105:0/596955160 shutdown_connections 2024-09-21T01:44:38.578 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.577+0000 7f4fb677d640 1 --2- 172.21.15.105:0/596955160 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4fb0131860 0x7f4fb0133c50 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.578 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.577+0000 7f4fb677d640 1 --2- 172.21.15.105:0/596955160 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4fb009a820 0x7f4fb0131320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.578 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.577+0000 7f4fb677d640 1 -- 172.21.15.105:0/596955160 >> 172.21.15.105:0/596955160 conn(0x7f4fb009b760 msgr2=0x7f4fb009bb70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:38.578 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.577+0000 7f4fb677d640 1 -- 172.21.15.105:0/596955160 shutdown_connections 2024-09-21T01:44:38.579 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.577+0000 7f4fb677d640 1 -- 172.21.15.105:0/596955160 wait complete. 2024-09-21T01:44:38.581 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.578+0000 7f4fb677d640 1 Processor -- start 2024-09-21T01:44:38.581 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.578+0000 7f4fb677d640 1 -- start start 2024-09-21T01:44:38.581 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.579+0000 7f4fb677d640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4fb009a820 0x7f4fb012d040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:38.581 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.579+0000 7f4fb677d640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4fb012c480 0x7f4fb012d580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:38.581 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.579+0000 7f4fb677d640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4fb0131860 0x7f4fb012dac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:38.581 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.579+0000 7f4fb677d640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4fb012f420 con 0x7f4fb0131860 2024-09-21T01:44:38.581 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.579+0000 7f4fb677d640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f4fb012f6d0 con 0x7f4fb012c480 2024-09-21T01:44:38.581 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.579+0000 7f4fb677d640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f4fb01267e0 con 0x7f4fb009a820 2024-09-21T01:44:38.581 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.579+0000 7f4faf7fe640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4fb012c480 0x7f4fb012d580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:38.581 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.579+0000 7f4fb4d1a640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4fb0131860 0x7f4fb012dac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:38.581 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.579+0000 7f4faffff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4fb009a820 0x7f4fb012d040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:38.582 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.579+0000 7f4fb4d1a640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4fb0131860 0x7f4fb012dac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:55506/0 (socket says 172.21.15.105:55506) 2024-09-21T01:44:38.582 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.579+0000 7f4fb4d1a640 1 -- 172.21.15.105:0/1066390930 learned_addr learned my addr 172.21.15.105:0/1066390930 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:38.582 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.579+0000 7f4faf7fe640 1 -- 172.21.15.105:0/1066390930 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4fb009a820 msgr2=0x7f4fb012d040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.582 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.579+0000 7f4faf7fe640 1 --2- 172.21.15.105:0/1066390930 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4fb009a820 0x7f4fb012d040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.582 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.579+0000 7f4faf7fe640 1 -- 172.21.15.105:0/1066390930 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4fb0131860 msgr2=0x7f4fb012dac0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.582 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.579+0000 7f4faf7fe640 1 --2- 172.21.15.105:0/1066390930 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4fb0131860 0x7f4fb012dac0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.582 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.579+0000 7f4faf7fe640 1 -- 172.21.15.105:0/1066390930 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4fa4009490 con 0x7f4fb012c480 2024-09-21T01:44:38.582 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.579+0000 7f4faffff640 1 --2- 172.21.15.105:0/1066390930 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4fb009a820 0x7f4fb012d040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:38.582 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.579+0000 7f4faf7fe640 1 --2- 172.21.15.105:0/1066390930 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4fb012c480 0x7f4fb012d580 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f4fa400e9f0 tx=0x7f4fa400eeb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:38.582 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.580+0000 7f4facff9640 1 -- 172.21.15.105:0/1066390930 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4fa400cd20 con 0x7f4fb012c480 2024-09-21T01:44:38.582 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.580+0000 7f4facff9640 1 -- 172.21.15.105:0/1066390930 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f4fa400ce80 con 0x7f4fb012c480 2024-09-21T01:44:38.582 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.580+0000 7f4facff9640 1 -- 172.21.15.105:0/1066390930 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4fa4004240 con 0x7f4fb012c480 2024-09-21T01:44:38.582 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.580+0000 7f4fb677d640 1 -- 172.21.15.105:0/1066390930 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4fa8009840 con 0x7f4fb012c480 2024-09-21T01:44:38.582 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.580+0000 7f4fb677d640 1 -- 172.21.15.105:0/1066390930 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4fb012e980 con 0x7f4fb012c480 2024-09-21T01:44:38.583 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.580+0000 7f4facff9640 1 -- 172.21.15.105:0/1066390930 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f4fa4005020 con 0x7f4fb012c480 2024-09-21T01:44:38.583 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.580+0000 7f4facff9640 1 -- 172.21.15.105:0/1066390930 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 725+0+0 (secure 0 0 0) 0x7f4fa4014070 con 0x7f4fb012c480 2024-09-21T01:44:38.583 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.581+0000 7f4f967fc640 1 -- 172.21.15.105:0/1066390930 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4f88005250 con 0x7f4fb012c480 2024-09-21T01:44:38.583 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.583+0000 7f4facff9640 1 -- 172.21.15.105:0/1066390930 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f4fa40043a0 con 0x7f4fb012c480 2024-09-21T01:44:38.645 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.645+0000 7f4f967fc640 1 -- 172.21.15.105:0/1066390930 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "osd crush tunables", "profile": "default"} v 0) v1 -- 0x7f4f880057d0 con 0x7f4fb012c480 2024-09-21T01:44:38.649 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.648+0000 7f4facff9640 1 -- 172.21.15.105:0/1066390930 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{"prefix": "osd crush tunables", "profile": "default"}]=0 adjusted tunables profile to default v2) v1 ==== 124+0+0 (secure 0 0 0) 0x7f4fa4005500 con 0x7f4fb012c480 2024-09-21T01:44:38.649 INFO:teuthology.orchestra.run.smithi105.stderr:adjusted tunables profile to default 2024-09-21T01:44:38.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.649+0000 7f4f967fc640 1 -- 172.21.15.105:0/1066390930 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4fb012c480 msgr2=0x7f4fb012d580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:38.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.649+0000 7f4f967fc640 1 --2- 172.21.15.105:0/1066390930 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4fb012c480 0x7f4fb012d580 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f4fa400e9f0 tx=0x7f4fa400eeb0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.649+0000 7f4f967fc640 1 -- 172.21.15.105:0/1066390930 shutdown_connections 2024-09-21T01:44:38.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.649+0000 7f4f967fc640 1 --2- 172.21.15.105:0/1066390930 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4fb0131860 0x7f4fb012dac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.649+0000 7f4f967fc640 1 --2- 172.21.15.105:0/1066390930 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4fb012c480 0x7f4fb012d580 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.649+0000 7f4f967fc640 1 --2- 172.21.15.105:0/1066390930 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4fb009a820 0x7f4fb012d040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:38.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.649+0000 7f4f967fc640 1 -- 172.21.15.105:0/1066390930 >> 172.21.15.105:0/1066390930 conn(0x7f4fb009b760 msgr2=0x7f4fb012abb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:38.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.649+0000 7f4f967fc640 1 -- 172.21.15.105:0/1066390930 shutdown_connections 2024-09-21T01:44:38.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:38.649+0000 7f4f967fc640 1 -- 172.21.15.105:0/1066390930 wait complete. 2024-09-21T01:44:38.659 INFO:tasks.ceph:check_enable_crimson: False 2024-09-21T01:44:38.660 INFO:tasks.ceph:Starting osd daemons in cluster ceph... 2024-09-21T01:44:38.660 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:44:38.660 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/var/lib/ceph/osd/ceph-0/fsid of=/dev/stdout 2024-09-21T01:44:38.663 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:38.663+0000 7f43c91dd040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T01:44:38.663 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:38.663+0000 7fc75c220040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T01:44:38.686 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:44:38.686 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/var/lib/ceph/osd/ceph-1/fsid of=/dev/stdout 2024-09-21T01:44:38.722 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:38.721+0000 7f43c91dd040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T01:44:38.722 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:38.721+0000 7fc75c220040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T01:44:38.746 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:44:38.746 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/var/lib/ceph/osd/ceph-2/fsid of=/dev/stdout 2024-09-21T01:44:38.811 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:44:38.811 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/var/lib/ceph/osd/ceph-3/fsid of=/dev/stdout 2024-09-21T01:44:38.866 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:38.865+0000 7f43c91dd040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T01:44:38.866 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:38.866+0000 7fc75c220040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T01:44:38.875 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph osd new 03218bdb-ec48-44c7-bbf5-31cb53a3cea7 0 2024-09-21T01:44:39.016 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.016+0000 7f080751f640 1 -- 172.21.15.105:0/3182411494 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0800128fc0 msgr2=0x7f0800129430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:39.016 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.016+0000 7f080751f640 1 --2- 172.21.15.105:0/3182411494 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0800128fc0 0x7f0800129430 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f07f8005fd0 tx=0x7f07f802a8e0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.017 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.016+0000 7f080751f640 1 -- 172.21.15.105:0/3182411494 shutdown_connections 2024-09-21T01:44:39.017 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.016+0000 7f080751f640 1 --2- 172.21.15.105:0/3182411494 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0800128fc0 0x7f0800129430 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.017 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.016+0000 7f080751f640 1 --2- 172.21.15.105:0/3182411494 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f080009a820 0x7f0800128a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.017 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.016+0000 7f080751f640 1 --2- 172.21.15.105:0/3182411494 >> v2:172.21.15.105:3300/0 conn(0x7f080012c480 0x7f080012c860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.017 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.016+0000 7f080751f640 1 -- 172.21.15.105:0/3182411494 >> 172.21.15.105:0/3182411494 conn(0x7f080009b760 msgr2=0x7f080009bb70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:39.017 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.016+0000 7f080751f640 1 -- 172.21.15.105:0/3182411494 shutdown_connections 2024-09-21T01:44:39.017 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.016+0000 7f080751f640 1 -- 172.21.15.105:0/3182411494 wait complete. 2024-09-21T01:44:39.020 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.017+0000 7f080751f640 1 Processor -- start 2024-09-21T01:44:39.020 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.017+0000 7f080751f640 1 -- start start 2024-09-21T01:44:39.020 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.017+0000 7f080751f640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f080009a820 0x7f0800121fc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:39.020 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.017+0000 7f080751f640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0800128fc0 0x7f0800122500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:39.020 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.017+0000 7f080751f640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f080012c480 0x7f080012ff10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:39.020 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.017+0000 7f080751f640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0800115740 con 0x7f080012c480 2024-09-21T01:44:39.020 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.017+0000 7f080751f640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0800115370 con 0x7f080009a820 2024-09-21T01:44:39.020 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.017+0000 7f080751f640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f0800131970 con 0x7f0800128fc0 2024-09-21T01:44:39.020 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.017+0000 7f0804aba640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0800128fc0 0x7f0800122500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:39.020 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.017+0000 7f08052bb640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f080009a820 0x7f0800121fc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:39.020 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.017+0000 7f08052bb640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f080009a820 0x7f0800121fc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:50386/0 (socket says 172.21.15.105:50386) 2024-09-21T01:44:39.020 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.017+0000 7f0804aba640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0800128fc0 0x7f0800122500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:45638/0 (socket says 172.21.15.105:45638) 2024-09-21T01:44:39.020 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.017+0000 7f08052bb640 1 -- 172.21.15.105:0/3379940188 learned_addr learned my addr 172.21.15.105:0/3379940188 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:39.021 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.017+0000 7f0805abc640 1 --2- 172.21.15.105:0/3379940188 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f080012c480 0x7f080012ff10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:39.021 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.017+0000 7f0804aba640 1 -- 172.21.15.105:0/3379940188 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f080009a820 msgr2=0x7f0800121fc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:39.021 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.017+0000 7f0804aba640 1 --2- 172.21.15.105:0/3379940188 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f080009a820 0x7f0800121fc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.021 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.017+0000 7f0804aba640 1 -- 172.21.15.105:0/3379940188 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f080012c480 msgr2=0x7f080012ff10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:39.021 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.017+0000 7f0804aba640 1 --2- 172.21.15.105:0/3379940188 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f080012c480 0x7f080012ff10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.021 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.017+0000 7f0804aba640 1 -- 172.21.15.105:0/3379940188 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f07fc009490 con 0x7f0800128fc0 2024-09-21T01:44:39.021 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.018+0000 7f08052bb640 1 --2- 172.21.15.105:0/3379940188 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f080009a820 0x7f0800121fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:39.021 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.018+0000 7f0804aba640 1 --2- 172.21.15.105:0/3379940188 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0800128fc0 0x7f0800122500 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f07f000e7c0 tx=0x7f07f000ec80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:39.021 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.018+0000 7f07f5ffb640 1 -- 172.21.15.105:0/3379940188 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f07f000ccc0 con 0x7f0800128fc0 2024-09-21T01:44:39.021 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.018+0000 7f07f5ffb640 1 -- 172.21.15.105:0/3379940188 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f07f000eed0 con 0x7f0800128fc0 2024-09-21T01:44:39.021 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.018+0000 7f080751f640 1 -- 172.21.15.105:0/3379940188 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f07f8009900 con 0x7f0800128fc0 2024-09-21T01:44:39.021 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.018+0000 7f07f5ffb640 1 -- 172.21.15.105:0/3379940188 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f07f0004280 con 0x7f0800128fc0 2024-09-21T01:44:39.021 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.018+0000 7f080751f640 1 -- 172.21.15.105:0/3379940188 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f08001321c0 con 0x7f0800128fc0 2024-09-21T01:44:39.021 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.018+0000 7f07f5ffb640 1 -- 172.21.15.105:0/3379940188 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f07f0005020 con 0x7f0800128fc0 2024-09-21T01:44:39.021 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.018+0000 7f07f5ffb640 1 -- 172.21.15.105:0/3379940188 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 725+0+0 (secure 0 0 0) 0x7f07f0014070 con 0x7f0800128fc0 2024-09-21T01:44:39.022 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.019+0000 7f080751f640 1 -- 172.21.15.105:0/3379940188 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f080012b460 con 0x7f0800128fc0 2024-09-21T01:44:39.022 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.021+0000 7f07f5ffb640 1 -- 172.21.15.105:0/3379940188 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f07f001c070 con 0x7f0800128fc0 2024-09-21T01:44:39.090 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.090+0000 7f080751f640 1 -- 172.21.15.105:0/3379940188 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd new", "uuid": "03218bdb-ec48-44c7-bbf5-31cb53a3cea7", "id": 0} v 0) v1 -- 0x7f080012a760 con 0x7f0800128fc0 2024-09-21T01:44:39.094 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.094+0000 7f07f5ffb640 1 -- 172.21.15.105:0/3379940188 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "03218bdb-ec48-44c7-bbf5-31cb53a3cea7", "id": 0}]=0 v3) v1 ==== 112+0+2 (secure 0 0 0) 0x7f07f00043e0 con 0x7f0800128fc0 2024-09-21T01:44:39.094 INFO:teuthology.orchestra.run.smithi105.stdout:0 2024-09-21T01:44:39.095 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.095+0000 7f080751f640 1 -- 172.21.15.105:0/3379940188 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0800128fc0 msgr2=0x7f0800122500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:39.096 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.095+0000 7f080751f640 1 --2- 172.21.15.105:0/3379940188 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0800128fc0 0x7f0800122500 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f07f000e7c0 tx=0x7f07f000ec80 comp rx=0 tx=0).stop 2024-09-21T01:44:39.096 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.095+0000 7f080751f640 1 -- 172.21.15.105:0/3379940188 shutdown_connections 2024-09-21T01:44:39.096 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.095+0000 7f080751f640 1 --2- 172.21.15.105:0/3379940188 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f080012c480 0x7f080012ff10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.096 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.095+0000 7f080751f640 1 --2- 172.21.15.105:0/3379940188 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0800128fc0 0x7f0800122500 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.096 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.095+0000 7f080751f640 1 --2- 172.21.15.105:0/3379940188 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f080009a820 0x7f0800121fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.096 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.095+0000 7f080751f640 1 -- 172.21.15.105:0/3379940188 >> 172.21.15.105:0/3379940188 conn(0x7f080009b760 msgr2=0x7f080009c980 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:39.096 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.095+0000 7f080751f640 1 -- 172.21.15.105:0/3379940188 shutdown_connections 2024-09-21T01:44:39.096 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.095+0000 7f080751f640 1 -- 172.21.15.105:0/3379940188 wait complete. 2024-09-21T01:44:39.106 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph osd new 5528ff44-922a-4e44-bb69-95fd9544d8dd 1 2024-09-21T01:44:39.196 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:39.195+0000 7f43c91dd040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T01:44:39.196 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:39.195+0000 7fc75c220040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T01:44:39.206 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.205+0000 7f10ff79c640 1 -- 172.21.15.105:0/1842922769 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f10f809a820 msgr2=0x7f10f8128a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:39.206 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.205+0000 7f10ff79c640 1 --2- 172.21.15.105:0/1842922769 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f10f809a820 0x7f10f8128a80 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f10e8009860 tx=0x7f10e802a3d0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.206 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.205+0000 7f10ff79c640 1 -- 172.21.15.105:0/1842922769 shutdown_connections 2024-09-21T01:44:39.206 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.205+0000 7f10ff79c640 1 --2- 172.21.15.105:0/1842922769 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f10f809a820 0x7f10f8128a80 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.206 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.205+0000 7f10ff79c640 1 --2- 172.21.15.105:0/1842922769 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f10f812c480 0x7f10f812c860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.206 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.205+0000 7f10ff79c640 1 -- 172.21.15.105:0/1842922769 >> 172.21.15.105:0/1842922769 conn(0x7f10f809b760 msgr2=0x7f10f809bb70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:39.206 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.205+0000 7f10ff79c640 1 -- 172.21.15.105:0/1842922769 shutdown_connections 2024-09-21T01:44:39.206 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.205+0000 7f10ff79c640 1 -- 172.21.15.105:0/1842922769 wait complete. 2024-09-21T01:44:39.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.206+0000 7f10ff79c640 1 Processor -- start 2024-09-21T01:44:39.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.206+0000 7f10ff79c640 1 -- start start 2024-09-21T01:44:39.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.206+0000 7f10ff79c640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f10f809a820 0x7f10f8130010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:39.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.206+0000 7f10ff79c640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f10f8128fc0 0x7f10f8130550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:39.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.206+0000 7f10ff79c640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f10f812c480 0x7f10f8130a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:39.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.206+0000 7f10ff79c640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f10f81324a0 con 0x7f10f809a820 2024-09-21T01:44:39.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.206+0000 7f10ff79c640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f10f8126770 con 0x7f10f8128fc0 2024-09-21T01:44:39.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.206+0000 7f10ff79c640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f10f8133770 con 0x7f10f812c480 2024-09-21T01:44:39.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.207+0000 7f10fd538640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f10f809a820 0x7f10f8130010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:39.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.207+0000 7f10fdd39640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f10f812c480 0x7f10f8130a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:39.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.207+0000 7f10fd538640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f10f809a820 0x7f10f8130010 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:55528/0 (socket says 172.21.15.105:55528) 2024-09-21T01:44:39.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.207+0000 7f10fd538640 1 -- 172.21.15.105:0/2350967554 learned_addr learned my addr 172.21.15.105:0/2350967554 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:39.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.207+0000 7f10fdd39640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f10f812c480 0x7f10f8130a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:45664/0 (socket says 172.21.15.105:45664) 2024-09-21T01:44:39.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.207+0000 7f10fcd37640 1 --2- 172.21.15.105:0/2350967554 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f10f8128fc0 0x7f10f8130550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:39.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.207+0000 7f10fd538640 1 -- 172.21.15.105:0/2350967554 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f10f812c480 msgr2=0x7f10f8130a90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:39.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.207+0000 7f10fd538640 1 --2- 172.21.15.105:0/2350967554 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f10f812c480 0x7f10f8130a90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.207+0000 7f10fd538640 1 -- 172.21.15.105:0/2350967554 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f10f8128fc0 msgr2=0x7f10f8130550 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:39.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.207+0000 7f10fd538640 1 --2- 172.21.15.105:0/2350967554 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f10f8128fc0 0x7f10f8130550 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.207+0000 7f10fd538640 1 -- 172.21.15.105:0/2350967554 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f10f4009490 con 0x7f10f809a820 2024-09-21T01:44:39.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.207+0000 7f10fcd37640 1 --2- 172.21.15.105:0/2350967554 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f10f8128fc0 0x7f10f8130550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:39.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.207+0000 7f10fd538640 1 --2- 172.21.15.105:0/2350967554 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f10f809a820 0x7f10f8130010 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7f10f400ea60 tx=0x7f10f400ef20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:39.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.207+0000 7f10edffb640 1 -- 172.21.15.105:0/2350967554 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f10f400cde0 con 0x7f10f809a820 2024-09-21T01:44:39.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.207+0000 7f10edffb640 1 -- 172.21.15.105:0/2350967554 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f10f4004020 con 0x7f10f809a820 2024-09-21T01:44:39.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.207+0000 7f10ff79c640 1 -- 172.21.15.105:0/2350967554 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f10e8009490 con 0x7f10f809a820 2024-09-21T01:44:39.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.207+0000 7f10edffb640 1 -- 172.21.15.105:0/2350967554 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f10f4004410 con 0x7f10f809a820 2024-09-21T01:44:39.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.208+0000 7f10ff79c640 1 -- 172.21.15.105:0/2350967554 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f10f8131a70 con 0x7f10f809a820 2024-09-21T01:44:39.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.208+0000 7f10edffb640 1 -- 172.21.15.105:0/2350967554 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f10f40035f0 con 0x7f10f809a820 2024-09-21T01:44:39.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.208+0000 7f10edffb640 1 -- 172.21.15.105:0/2350967554 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 840+0+0 (secure 0 0 0) 0x7f10f4014070 con 0x7f10f809a820 2024-09-21T01:44:39.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.209+0000 7f10ff79c640 1 -- 172.21.15.105:0/2350967554 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f10f8128a80 con 0x7f10f809a820 2024-09-21T01:44:39.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.211+0000 7f10edffb640 1 -- 172.21.15.105:0/2350967554 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f10f4003c50 con 0x7f10f809a820 2024-09-21T01:44:39.249 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:39.249+0000 7f43c91dd040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T01:44:39.250 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:39.249+0000 7fc75c220040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T01:44:39.279 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.279+0000 7f10ff79c640 1 -- 172.21.15.105:0/2350967554 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd new", "uuid": "5528ff44-922a-4e44-bb69-95fd9544d8dd", "id": 1} v 0) v1 -- 0x7f10f806afb0 con 0x7f10f809a820 2024-09-21T01:44:39.282 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.282+0000 7f10edffb640 1 -- 172.21.15.105:0/2350967554 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "5528ff44-922a-4e44-bb69-95fd9544d8dd", "id": 1}]=0 v4) v1 ==== 112+0+2 (secure 0 0 0) 0x7f10f4003df0 con 0x7f10f809a820 2024-09-21T01:44:39.283 INFO:teuthology.orchestra.run.smithi105.stdout:1 2024-09-21T01:44:39.284 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.283+0000 7f10ff79c640 1 -- 172.21.15.105:0/2350967554 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f10f809a820 msgr2=0x7f10f8130010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:39.284 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.283+0000 7f10ff79c640 1 --2- 172.21.15.105:0/2350967554 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f10f809a820 0x7f10f8130010 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7f10f400ea60 tx=0x7f10f400ef20 comp rx=0 tx=0).stop 2024-09-21T01:44:39.284 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.283+0000 7f10ff79c640 1 -- 172.21.15.105:0/2350967554 shutdown_connections 2024-09-21T01:44:39.284 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.283+0000 7f10ff79c640 1 --2- 172.21.15.105:0/2350967554 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f10f812c480 0x7f10f8130a90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.284 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.283+0000 7f10ff79c640 1 --2- 172.21.15.105:0/2350967554 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f10f8128fc0 0x7f10f8130550 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.284 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.283+0000 7f10ff79c640 1 --2- 172.21.15.105:0/2350967554 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f10f809a820 0x7f10f8130010 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.284 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.283+0000 7f10ff79c640 1 -- 172.21.15.105:0/2350967554 >> 172.21.15.105:0/2350967554 conn(0x7f10f809b760 msgr2=0x7f10f809d3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:39.284 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.283+0000 7f10ff79c640 1 -- 172.21.15.105:0/2350967554 shutdown_connections 2024-09-21T01:44:39.284 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.284+0000 7f10ff79c640 1 -- 172.21.15.105:0/2350967554 wait complete. 2024-09-21T01:44:39.296 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph osd new df56ff35-71e9-4d46-8abb-84265eb6e1ee 2 2024-09-21T01:44:39.340 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:39.340+0000 7fc75c220040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T01:44:39.341 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:39.341+0000 7f43c91dd040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T01:44:39.397 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.397+0000 7f42ada5a640 1 -- 172.21.15.105:0/2923411550 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f42a809a820 msgr2=0x7f42a8128a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:39.398 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.397+0000 7f42ada5a640 1 --2- 172.21.15.105:0/2923411550 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f42a809a820 0x7f42a8128a80 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f4290009900 tx=0x7f429002a460 comp rx=0 tx=0).stop 2024-09-21T01:44:39.398 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.397+0000 7f42ada5a640 1 -- 172.21.15.105:0/2923411550 shutdown_connections 2024-09-21T01:44:39.398 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.397+0000 7f42ada5a640 1 --2- 172.21.15.105:0/2923411550 >> v2:172.21.15.105:3300/0 conn(0x7f42a8128fc0 0x7f42a8129450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.398 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.397+0000 7f42ada5a640 1 --2- 172.21.15.105:0/2923411550 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f42a809a820 0x7f42a8128a80 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.398 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.397+0000 7f42ada5a640 1 --2- 172.21.15.105:0/2923411550 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f42a812c480 0x7f42a812c860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.398 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.397+0000 7f42ada5a640 1 -- 172.21.15.105:0/2923411550 >> 172.21.15.105:0/2923411550 conn(0x7f42a809b760 msgr2=0x7f42a809bb70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:39.398 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.397+0000 7f42ada5a640 1 -- 172.21.15.105:0/2923411550 shutdown_connections 2024-09-21T01:44:39.398 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.397+0000 7f42ada5a640 1 -- 172.21.15.105:0/2923411550 wait complete. 2024-09-21T01:44:39.399 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.398+0000 7f42ada5a640 1 Processor -- start 2024-09-21T01:44:39.399 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.398+0000 7f42ada5a640 1 -- start start 2024-09-21T01:44:39.400 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.399+0000 7f42ada5a640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f42a809a820 0x7f42a81c21f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:39.401 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.399+0000 7f42ada5a640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f42a8128fc0 0x7f42a81c2730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:39.401 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.399+0000 7f42ada5a640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f42a812c480 0x7f42a812fc10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:39.401 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.399+0000 7f42ada5a640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f42a81328f0 con 0x7f42a809a820 2024-09-21T01:44:39.401 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.399+0000 7f42ada5a640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f42a8115480 con 0x7f42a812c480 2024-09-21T01:44:39.401 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.399+0000 7f42ada5a640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f42a8115170 con 0x7f42a8128fc0 2024-09-21T01:44:39.401 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.399+0000 7f429ffff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f42a8128fc0 0x7f42a81c2730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:39.401 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.399+0000 7f429ffff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f42a8128fc0 0x7f42a81c2730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:45684/0 (socket says 172.21.15.105:45684) 2024-09-21T01:44:39.401 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.399+0000 7f429ffff640 1 -- 172.21.15.105:0/4036443612 learned_addr learned my addr 172.21.15.105:0/4036443612 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:39.401 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.399+0000 7f42a6ffd640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f42a809a820 0x7f42a81c21f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:39.401 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.399+0000 7f42a77fe640 1 --2- 172.21.15.105:0/4036443612 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f42a812c480 0x7f42a812fc10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:39.401 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.399+0000 7f429ffff640 1 -- 172.21.15.105:0/4036443612 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f42a812c480 msgr2=0x7f42a812fc10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:39.401 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.399+0000 7f429ffff640 1 --2- 172.21.15.105:0/4036443612 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f42a812c480 0x7f42a812fc10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.401 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.399+0000 7f429ffff640 1 -- 172.21.15.105:0/4036443612 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f42a809a820 msgr2=0x7f42a81c21f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:39.401 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.399+0000 7f429ffff640 1 --2- 172.21.15.105:0/4036443612 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f42a809a820 0x7f42a81c21f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.399+0000 7f429ffff640 1 -- 172.21.15.105:0/4036443612 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f42a0009840 con 0x7f42a8128fc0 2024-09-21T01:44:39.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.399+0000 7f42a6ffd640 1 --2- 172.21.15.105:0/4036443612 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f42a809a820 0x7f42a81c21f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T01:44:39.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.399+0000 7f429ffff640 1 --2- 172.21.15.105:0/4036443612 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f42a8128fc0 0x7f42a81c2730 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f4290009850 tx=0x7f4290003820 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:39.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.400+0000 7f42a4ff9640 1 -- 172.21.15.105:0/4036443612 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4290003e20 con 0x7f42a8128fc0 2024-09-21T01:44:39.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.400+0000 7f42a4ff9640 1 -- 172.21.15.105:0/4036443612 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f429003c470 con 0x7f42a8128fc0 2024-09-21T01:44:39.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.400+0000 7f42ada5a640 1 -- 172.21.15.105:0/4036443612 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4298009490 con 0x7f42a8128fc0 2024-09-21T01:44:39.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.400+0000 7f42a4ff9640 1 -- 172.21.15.105:0/4036443612 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f429003c800 con 0x7f42a8128fc0 2024-09-21T01:44:39.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.400+0000 7f42ada5a640 1 -- 172.21.15.105:0/4036443612 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4290009560 con 0x7f42a8128fc0 2024-09-21T01:44:39.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.400+0000 7f42a4ff9640 1 -- 172.21.15.105:0/4036443612 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f429003cc60 con 0x7f42a8128fc0 2024-09-21T01:44:39.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.400+0000 7f42a4ff9640 1 -- 172.21.15.105:0/4036443612 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 955+0+0 (secure 0 0 0) 0x7f4290045d50 con 0x7f42a8128fc0 2024-09-21T01:44:39.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.401+0000 7f42ada5a640 1 -- 172.21.15.105:0/4036443612 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f42a812b660 con 0x7f42a8128fc0 2024-09-21T01:44:39.404 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.404+0000 7f42a4ff9640 1 -- 172.21.15.105:0/4036443612 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f429004a060 con 0x7f42a8128fc0 2024-09-21T01:44:39.438 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:39.438+0000 7f43c91dd040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T01:44:39.440 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:39.440+0000 7fc75c220040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T01:44:39.473 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.473+0000 7f42ada5a640 1 -- 172.21.15.105:0/4036443612 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd new", "uuid": "df56ff35-71e9-4d46-8abb-84265eb6e1ee", "id": 2} v 0) v1 -- 0x7f42a8129450 con 0x7f42a8128fc0 2024-09-21T01:44:39.476 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.476+0000 7f42a4ff9640 1 -- 172.21.15.105:0/4036443612 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "df56ff35-71e9-4d46-8abb-84265eb6e1ee", "id": 2}]=0 v5) v1 ==== 112+0+2 (secure 0 0 0) 0x7f4290040070 con 0x7f42a8128fc0 2024-09-21T01:44:39.477 INFO:teuthology.orchestra.run.smithi105.stdout:2 2024-09-21T01:44:39.478 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.477+0000 7f42ada5a640 1 -- 172.21.15.105:0/4036443612 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f42a8128fc0 msgr2=0x7f42a81c2730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:39.478 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.477+0000 7f42ada5a640 1 --2- 172.21.15.105:0/4036443612 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f42a8128fc0 0x7f42a81c2730 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f4290009850 tx=0x7f4290003820 comp rx=0 tx=0).stop 2024-09-21T01:44:39.478 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.477+0000 7f42ada5a640 1 -- 172.21.15.105:0/4036443612 shutdown_connections 2024-09-21T01:44:39.478 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.477+0000 7f42ada5a640 1 --2- 172.21.15.105:0/4036443612 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f42a812c480 0x7f42a812fc10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.478 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.477+0000 7f42ada5a640 1 --2- 172.21.15.105:0/4036443612 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f42a8128fc0 0x7f42a81c2730 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.478 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.477+0000 7f42ada5a640 1 --2- 172.21.15.105:0/4036443612 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f42a809a820 0x7f42a81c21f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.478 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.477+0000 7f42ada5a640 1 -- 172.21.15.105:0/4036443612 >> 172.21.15.105:0/4036443612 conn(0x7f42a809b760 msgr2=0x7f42a809c980 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:39.478 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.477+0000 7f42ada5a640 1 -- 172.21.15.105:0/4036443612 shutdown_connections 2024-09-21T01:44:39.478 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.477+0000 7f42ada5a640 1 -- 172.21.15.105:0/4036443612 wait complete. 2024-09-21T01:44:39.487 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:39.487+0000 7f43c91dd040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T01:44:39.488 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph osd new 012d9668-f1d3-4268-b0eb-15fa845ef17c 3 2024-09-21T01:44:39.489 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:39.489+0000 7fc75c220040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T01:44:39.590 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.589+0000 7f8e93461640 1 -- 172.21.15.105:0/3939363329 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8e8c12a820 msgr2=0x7f8e8c126a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:39.590 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.589+0000 7f8e93461640 1 --2- 172.21.15.105:0/3939363329 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8e8c12a820 0x7f8e8c126a00 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7f8e7c005fd0 tx=0x7f8e7c02a420 comp rx=0 tx=0).stop 2024-09-21T01:44:39.590 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.589+0000 7f8e93461640 1 -- 172.21.15.105:0/3939363329 shutdown_connections 2024-09-21T01:44:39.590 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.589+0000 7f8e93461640 1 --2- 172.21.15.105:0/3939363329 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8e8c126f40 0x7f8e8c1273b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.590 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.589+0000 7f8e93461640 1 --2- 172.21.15.105:0/3939363329 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8e8c12a820 0x7f8e8c126a00 unknown :-1 s=CLOSED pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.590 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.589+0000 7f8e93461640 1 --2- 172.21.15.105:0/3939363329 >> v2:172.21.15.105:3300/0 conn(0x7f8e8c129f00 0x7f8e8c12a2e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.590 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.589+0000 7f8e93461640 1 -- 172.21.15.105:0/3939363329 >> 172.21.15.105:0/3939363329 conn(0x7f8e8c09ca60 msgr2=0x7f8e8c09ce70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:39.590 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.589+0000 7f8e93461640 1 -- 172.21.15.105:0/3939363329 shutdown_connections 2024-09-21T01:44:39.591 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.589+0000 7f8e93461640 1 -- 172.21.15.105:0/3939363329 wait complete. 2024-09-21T01:44:39.594 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.590+0000 7f8e93461640 1 Processor -- start 2024-09-21T01:44:39.594 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.590+0000 7f8e93461640 1 -- start start 2024-09-21T01:44:39.594 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.591+0000 7f8e93461640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8e8c126f40 0x7f8e8c0957e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:39.594 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.591+0000 7f8e93461640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8e8c129f00 0x7f8e8c095d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:39.594 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.591+0000 7f8e93461640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8e8c12a820 0x7f8e8c096260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:39.594 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.591+0000 7f8e93461640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e8c130240 con 0x7f8e8c126f40 2024-09-21T01:44:39.594 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.591+0000 7f8e93461640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f8e8c071880 con 0x7f8e8c12a820 2024-09-21T01:44:39.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.591+0000 7f8e93461640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f8e8c12f810 con 0x7f8e8c129f00 2024-09-21T01:44:39.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.591+0000 7f8e909fc640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8e8c129f00 0x7f8e8c095d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:39.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.591+0000 7f8e919fe640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8e8c12a820 0x7f8e8c096260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:39.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.591+0000 7f8e909fc640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8e8c129f00 0x7f8e8c095d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:45708/0 (socket says 172.21.15.105:45708) 2024-09-21T01:44:39.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.591+0000 7f8e909fc640 1 -- 172.21.15.105:0/253969681 learned_addr learned my addr 172.21.15.105:0/253969681 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:39.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.591+0000 7f8e911fd640 1 --2- 172.21.15.105:0/253969681 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8e8c126f40 0x7f8e8c0957e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:39.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.591+0000 7f8e909fc640 1 -- 172.21.15.105:0/253969681 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8e8c12a820 msgr2=0x7f8e8c096260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:39.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.591+0000 7f8e909fc640 1 --2- 172.21.15.105:0/253969681 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8e8c12a820 0x7f8e8c096260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.591+0000 7f8e909fc640 1 -- 172.21.15.105:0/253969681 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8e8c126f40 msgr2=0x7f8e8c0957e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:39.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.591+0000 7f8e909fc640 1 --2- 172.21.15.105:0/253969681 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8e8c126f40 0x7f8e8c0957e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.591+0000 7f8e909fc640 1 -- 172.21.15.105:0/253969681 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8e7c009560 con 0x7f8e8c129f00 2024-09-21T01:44:39.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.591+0000 7f8e909fc640 1 --2- 172.21.15.105:0/253969681 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8e8c129f00 0x7f8e8c095d20 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7f8e7c02a940 tx=0x7f8e7c0038a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:39.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.591+0000 7f8e81ffb640 1 -- 172.21.15.105:0/253969681 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e7c003ea0 con 0x7f8e8c129f00 2024-09-21T01:44:39.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.591+0000 7f8e81ffb640 1 -- 172.21.15.105:0/253969681 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f8e7c005d50 con 0x7f8e8c129f00 2024-09-21T01:44:39.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.592+0000 7f8e93461640 1 -- 172.21.15.105:0/253969681 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8e8c130960 con 0x7f8e8c129f00 2024-09-21T01:44:39.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.592+0000 7f8e81ffb640 1 -- 172.21.15.105:0/253969681 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f8e7c03c6b0 con 0x7f8e8c129f00 2024-09-21T01:44:39.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.592+0000 7f8e93461640 1 -- 172.21.15.105:0/253969681 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8e8c130010 con 0x7f8e8c129f00 2024-09-21T01:44:39.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.592+0000 7f8e81ffb640 1 -- 172.21.15.105:0/253969681 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f8e7c03cb10 con 0x7f8e8c129f00 2024-09-21T01:44:39.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.592+0000 7f8e81ffb640 1 -- 172.21.15.105:0/253969681 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1070+0+0 (secure 0 0 0) 0x7f8e7c045c20 con 0x7f8e8c129f00 2024-09-21T01:44:39.596 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:39.592+0000 7fc75c220040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T01:44:39.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.593+0000 7f8e93461640 1 -- 172.21.15.105:0/253969681 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8e8c1283b0 con 0x7f8e8c129f00 2024-09-21T01:44:39.597 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:39.593+0000 7f43c91dd040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T01:44:39.597 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.596+0000 7f8e81ffb640 1 -- 172.21.15.105:0/253969681 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f8e7c064e30 con 0x7f8e8c129f00 2024-09-21T01:44:39.648 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:39.647+0000 7f43c91dd040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T01:44:39.648 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:39.647+0000 7fc75c220040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T01:44:39.665 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.664+0000 7f8e93461640 1 -- 172.21.15.105:0/253969681 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd new", "uuid": "012d9668-f1d3-4268-b0eb-15fa845ef17c", "id": 3} v 0) v1 -- 0x7f8e8c09cef0 con 0x7f8e8c129f00 2024-09-21T01:44:39.668 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.668+0000 7f8e81ffb640 1 -- 172.21.15.105:0/253969681 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd new", "uuid": "012d9668-f1d3-4268-b0eb-15fa845ef17c", "id": 3}]=0 v6) v1 ==== 112+0+2 (secure 0 0 0) 0x7f8e7c053020 con 0x7f8e8c129f00 2024-09-21T01:44:39.668 INFO:teuthology.orchestra.run.smithi105.stdout:3 2024-09-21T01:44:39.669 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.669+0000 7f8e93461640 1 -- 172.21.15.105:0/253969681 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8e8c129f00 msgr2=0x7f8e8c095d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:39.669 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.669+0000 7f8e93461640 1 --2- 172.21.15.105:0/253969681 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8e8c129f00 0x7f8e8c095d20 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7f8e7c02a940 tx=0x7f8e7c0038a0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.669 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.669+0000 7f8e93461640 1 -- 172.21.15.105:0/253969681 shutdown_connections 2024-09-21T01:44:39.669 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.669+0000 7f8e93461640 1 --2- 172.21.15.105:0/253969681 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8e8c12a820 0x7f8e8c096260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.670 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.669+0000 7f8e93461640 1 --2- 172.21.15.105:0/253969681 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8e8c129f00 0x7f8e8c095d20 unknown :-1 s=CLOSED pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.670 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.669+0000 7f8e93461640 1 --2- 172.21.15.105:0/253969681 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8e8c126f40 0x7f8e8c0957e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.670 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.669+0000 7f8e93461640 1 -- 172.21.15.105:0/253969681 >> 172.21.15.105:0/253969681 conn(0x7f8e8c09ca60 msgr2=0x7f8e8c098150 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:39.670 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.669+0000 7f8e93461640 1 -- 172.21.15.105:0/253969681 shutdown_connections 2024-09-21T01:44:39.670 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.669+0000 7f8e93461640 1 -- 172.21.15.105:0/253969681 wait complete. 2024-09-21T01:44:39.679 INFO:tasks.ceph.osd.0:Restarting daemon 2024-09-21T01:44:39.679 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 0 2024-09-21T01:44:39.681 INFO:tasks.ceph.osd.0:Started 2024-09-21T01:44:39.681 INFO:tasks.ceph.osd.1:Restarting daemon 2024-09-21T01:44:39.682 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 1 2024-09-21T01:44:39.683 INFO:tasks.ceph.osd.1:Started 2024-09-21T01:44:39.683 INFO:tasks.ceph.osd.2:Restarting daemon 2024-09-21T01:44:39.684 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 2 2024-09-21T01:44:39.686 INFO:tasks.ceph.osd.2:Started 2024-09-21T01:44:39.686 INFO:tasks.ceph.osd.3:Restarting daemon 2024-09-21T01:44:39.686 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 3 2024-09-21T01:44:39.688 INFO:tasks.ceph.osd.3:Started 2024-09-21T01:44:39.688 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T01:44:39.701 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:39.701+0000 7f43c91dd040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T01:44:39.702 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:39.701+0000 7fc75c220040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T01:44:39.824 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.820+0000 7f2db6f45640 1 -- 172.21.15.105:0/1804506696 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2db01316f0 msgr2=0x7f2db0133ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:39.824 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.820+0000 7f2db6f45640 1 --2- 172.21.15.105:0/1804506696 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2db01316f0 0x7f2db0133ae0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f2da8005fd0 tx=0x7f2da802a780 comp rx=0 tx=0).stop 2024-09-21T01:44:39.824 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.821+0000 7f2db6f45640 1 -- 172.21.15.105:0/1804506696 shutdown_connections 2024-09-21T01:44:39.824 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.821+0000 7f2db6f45640 1 --2- 172.21.15.105:0/1804506696 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2db01316f0 0x7f2db0133ae0 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.824 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.821+0000 7f2db6f45640 1 --2- 172.21.15.105:0/1804506696 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2db009b7a0 0x7f2db01311b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.824 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.821+0000 7f2db6f45640 1 -- 172.21.15.105:0/1804506696 >> 172.21.15.105:0/1804506696 conn(0x7f2db006c0f0 msgr2=0x7f2db006c500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:39.824 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.821+0000 7f2db6f45640 1 -- 172.21.15.105:0/1804506696 shutdown_connections 2024-09-21T01:44:39.824 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.821+0000 7f2db6f45640 1 -- 172.21.15.105:0/1804506696 wait complete. 2024-09-21T01:44:39.824 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.821+0000 7f2db6f45640 1 Processor -- start 2024-09-21T01:44:39.825 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.821+0000 7f2db6f45640 1 -- start start 2024-09-21T01:44:39.825 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.822+0000 7f2db6f45640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2db009b7a0 0x7f2db012d1b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:39.825 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.822+0000 7f2db6f45640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2db012c310 0x7f2db012d6f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:39.825 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.822+0000 7f2db6f45640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2db01316f0 0x7f2db012dc30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:39.825 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.822+0000 7f2db6f45640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2db01315b0 con 0x7f2db009b7a0 2024-09-21T01:44:39.825 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.822+0000 7f2daffff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2db012c310 0x7f2db012d6f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:39.825 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.822+0000 7f2db4ce1640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2db009b7a0 0x7f2db012d1b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:39.825 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.822+0000 7f2db54e2640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2db01316f0 0x7f2db012dc30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:39.825 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.822+0000 7f2daffff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2db012c310 0x7f2db012d6f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:50502/0 (socket says 172.21.15.105:50502) 2024-09-21T01:44:39.825 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.822+0000 7f2daffff640 1 -- 172.21.15.105:0/1837602548 learned_addr learned my addr 172.21.15.105:0/1837602548 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:39.826 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.822+0000 7f2db6f45640 1 -- 172.21.15.105:0/1837602548 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2db0130250 con 0x7f2db012c310 2024-09-21T01:44:39.826 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.822+0000 7f2db4ce1640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2db009b7a0 0x7f2db012d1b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:55612/0 (socket says 172.21.15.105:55612) 2024-09-21T01:44:39.826 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.822+0000 7f2db6f45640 1 -- 172.21.15.105:0/1837602548 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f2db012fa70 con 0x7f2db01316f0 2024-09-21T01:44:39.826 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.823+0000 7f2daffff640 1 -- 172.21.15.105:0/1837602548 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2db01316f0 msgr2=0x7f2db012dc30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:39.826 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.823+0000 7f2daffff640 1 --2- 172.21.15.105:0/1837602548 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2db01316f0 0x7f2db012dc30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.826 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.823+0000 7f2daffff640 1 -- 172.21.15.105:0/1837602548 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2db009b7a0 msgr2=0x7f2db012d1b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:39.826 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.823+0000 7f2daffff640 1 --2- 172.21.15.105:0/1837602548 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2db009b7a0 0x7f2db012d1b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.826 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.823+0000 7f2daffff640 1 -- 172.21.15.105:0/1837602548 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d9c009560 con 0x7f2db012c310 2024-09-21T01:44:39.826 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.823+0000 7f2db54e2640 1 --2- 172.21.15.105:0/1837602548 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2db01316f0 0x7f2db012dc30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:39.826 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.823+0000 7f2daffff640 1 --2- 172.21.15.105:0/1837602548 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2db012c310 0x7f2db012d6f0 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f2d9c00e9c0 tx=0x7f2d9c00ee80 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:39.826 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.823+0000 7f2dad7fa640 1 -- 172.21.15.105:0/1837602548 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d9c00cde0 con 0x7f2db012c310 2024-09-21T01:44:39.826 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.823+0000 7f2dad7fa640 1 -- 172.21.15.105:0/1837602548 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f2d9c004020 con 0x7f2db012c310 2024-09-21T01:44:39.826 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.823+0000 7f2db6f45640 1 -- 172.21.15.105:0/1837602548 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2da8009840 con 0x7f2db012c310 2024-09-21T01:44:39.826 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.823+0000 7f2dad7fa640 1 -- 172.21.15.105:0/1837602548 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2d9c004410 con 0x7f2db012c310 2024-09-21T01:44:39.827 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.823+0000 7f2db6f45640 1 -- 172.21.15.105:0/1837602548 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2db012e170 con 0x7f2db012c310 2024-09-21T01:44:39.827 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.823+0000 7f2dad7fa640 1 -- 172.21.15.105:0/1837602548 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 1) v1 ==== 668+0+0 (secure 0 0 0) 0x7f2d9c0035f0 con 0x7f2db012c310 2024-09-21T01:44:39.827 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.823+0000 7f2dad7fa640 1 -- 172.21.15.105:0/1837602548 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(6..6 src has 1..6) v4 ==== 1185+0+0 (secure 0 0 0) 0x7f2d9c014070 con 0x7f2db012c310 2024-09-21T01:44:39.827 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.824+0000 7f2db6f45640 1 -- 172.21.15.105:0/1837602548 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2db012c6f0 con 0x7f2db012c310 2024-09-21T01:44:39.827 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.826+0000 7f2dad7fa640 1 -- 172.21.15.105:0/1837602548 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f2d9c0038a0 con 0x7f2db012c310 2024-09-21T01:44:39.842 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:39.842+0000 7f43c91dd040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T01:44:39.843 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:39.843+0000 7fc75c220040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T01:44:39.885 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.884+0000 7f2db6f45640 1 -- 172.21.15.105:0/1837602548 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f2db012f1e0 con 0x7f2db012c310 2024-09-21T01:44:39.885 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.885+0000 7f2dad7fa640 1 -- 172.21.15.105:0/1837602548 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v6) v1 ==== 74+0+4152 (secure 0 0 0) 0x7f2d9c003a40 con 0x7f2db012c310 2024-09-21T01:44:39.885 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:39.885 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":6,"fsid":"355399d3-6490-4ad0-ac52-d793648ff276","created":"2024-09-21T01:44:37.282024+0000","modified":"2024-09-21T01:44:39.666350+0000","last_up_change":"0.000000","last_in_change":"2024-09-21T01:44:39.666350+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"03218bdb-ec48-44c7-bbf5-31cb53a3cea7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"5528ff44-922a-4e44-bb69-95fd9544d8dd","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"df56ff35-71e9-4d46-8abb-84265eb6e1ee","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"012d9668-f1d3-4268-b0eb-15fa845ef17c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T01:44:39.886 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.886+0000 7f2db6f45640 1 -- 172.21.15.105:0/1837602548 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2db012c310 msgr2=0x7f2db012d6f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:39.886 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.886+0000 7f2db6f45640 1 --2- 172.21.15.105:0/1837602548 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2db012c310 0x7f2db012d6f0 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f2d9c00e9c0 tx=0x7f2d9c00ee80 comp rx=0 tx=0).stop 2024-09-21T01:44:39.887 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.886+0000 7f2db6f45640 1 -- 172.21.15.105:0/1837602548 shutdown_connections 2024-09-21T01:44:39.887 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.886+0000 7f2db6f45640 1 --2- 172.21.15.105:0/1837602548 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2db01316f0 0x7f2db012dc30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.887 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.886+0000 7f2db6f45640 1 --2- 172.21.15.105:0/1837602548 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2db012c310 0x7f2db012d6f0 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.887 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.886+0000 7f2db6f45640 1 --2- 172.21.15.105:0/1837602548 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2db009b7a0 0x7f2db012d1b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:39.887 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.886+0000 7f2db6f45640 1 -- 172.21.15.105:0/1837602548 >> 172.21.15.105:0/1837602548 conn(0x7f2db006c0f0 msgr2=0x7f2db009c630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:39.887 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.886+0000 7f2db6f45640 1 -- 172.21.15.105:0/1837602548 shutdown_connections 2024-09-21T01:44:39.887 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:39.886+0000 7f2db6f45640 1 -- 172.21.15.105:0/1837602548 wait complete. 2024-09-21T01:44:39.894 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:39.894+0000 7f43c91dd040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T01:44:39.894 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:39.894+0000 7fc75c220040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T01:44:39.897 INFO:tasks.ceph.ceph_manager.ceph:[] 2024-09-21T01:44:39.897 INFO:tasks.ceph:Waiting for OSDs to come up 2024-09-21T01:44:39.947 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:39.947+0000 7f43c91dd040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T01:44:39.947 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:39.947+0000 7fc75c220040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T01:44:40.016 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:40.016+0000 7f43c91dd040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T01:44:40.017 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:40.016+0000 7fc75c220040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T01:44:40.062 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:40.062+0000 7fc75c220040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T01:44:40.062 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:40.062+0000 7f43c91dd040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T01:44:40.063 INFO:tasks.ceph.osd.0.smithi105.stderr:2024-09-21T01:44:40.063+0000 7fbf67a9c640 -1 Falling back to public interface 2024-09-21T01:44:40.073 INFO:tasks.ceph.osd.2.smithi105.stderr:2024-09-21T01:44:40.073+0000 7f4efbf34640 -1 Falling back to public interface 2024-09-21T01:44:40.107 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:40.107+0000 7f43c91dd040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T01:44:40.107 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:40.107+0000 7fc75c220040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T01:44:40.300 DEBUG:teuthology.orchestra.run.smithi105:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-09-21T01:44:40.322 INFO:tasks.ceph.osd.3.smithi105.stderr:2024-09-21T01:44:40.322+0000 7fe28a145640 -1 Falling back to public interface 2024-09-21T01:44:40.326 INFO:tasks.ceph.osd.1.smithi105.stderr:2024-09-21T01:44:40.326+0000 7fcfba212640 -1 Falling back to public interface 2024-09-21T01:44:40.341 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:40.340+0000 7fc75c220040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T01:44:40.341 INFO:tasks.ceph.mgr.x.smithi105.stderr:2024-09-21T01:44:40.340+0000 7f43c91dd040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T01:44:40.400 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.399+0000 7f6dc0df4640 1 -- 172.21.15.105:0/1963713972 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6dbc06eb30 msgr2=0x7f6dbc06ef10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:40.400 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.399+0000 7f6dc0df4640 1 --2- 172.21.15.105:0/1963713972 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6dbc06eb30 0x7f6dbc06ef10 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f6dac009860 tx=0x7f6dac02a3d0 comp rx=0 tx=0).stop 2024-09-21T01:44:40.400 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.399+0000 7f6dc0df4640 1 -- 172.21.15.105:0/1963713972 shutdown_connections 2024-09-21T01:44:40.400 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.399+0000 7f6dc0df4640 1 --2- 172.21.15.105:0/1963713972 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6dbc06f450 0x7f6dbc12c070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:40.400 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.399+0000 7f6dc0df4640 1 --2- 172.21.15.105:0/1963713972 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6dbc06eb30 0x7f6dbc06ef10 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:40.400 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.399+0000 7f6dc0df4640 1 -- 172.21.15.105:0/1963713972 >> 172.21.15.105:0/1963713972 conn(0x7f6dbc09b5d0 msgr2=0x7f6dbc09b9e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:40.400 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.399+0000 7f6dc0df4640 1 -- 172.21.15.105:0/1963713972 shutdown_connections 2024-09-21T01:44:40.401 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.399+0000 7f6dc0df4640 1 -- 172.21.15.105:0/1963713972 wait complete. 2024-09-21T01:44:40.403 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.400+0000 7f6dc0df4640 1 Processor -- start 2024-09-21T01:44:40.403 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.400+0000 7f6dc0df4640 1 -- start start 2024-09-21T01:44:40.403 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.401+0000 7f6dc0df4640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6dbc06eb30 0x7f6dbc1300e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:40.403 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.401+0000 7f6dc0df4640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6dbc06f450 0x7f6dbc130620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:40.403 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.401+0000 7f6dc0df4640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6dbc12c5b0 0x7f6dbc130b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:40.403 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.401+0000 7f6dc0df4640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6dbc1324c0 con 0x7f6dbc06eb30 2024-09-21T01:44:40.403 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.401+0000 7f6dc0df4640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6dbc131b00 con 0x7f6dbc12c5b0 2024-09-21T01:44:40.403 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.401+0000 7f6dc0df4640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f6dbc071890 con 0x7f6dbc06f450 2024-09-21T01:44:40.404 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.401+0000 7f6db9d9b640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6dbc06f450 0x7f6dbc130620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:40.404 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.401+0000 7f6dba59c640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6dbc06eb30 0x7f6dbc1300e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:40.404 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.401+0000 7f6db9d9b640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6dbc06f450 0x7f6dbc130620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:45818/0 (socket says 172.21.15.105:45818) 2024-09-21T01:44:40.404 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.401+0000 7f6db9d9b640 1 -- 172.21.15.105:0/4152103686 learned_addr learned my addr 172.21.15.105:0/4152103686 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:40.404 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.401+0000 7f6dbad9d640 1 --2- 172.21.15.105:0/4152103686 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6dbc12c5b0 0x7f6dbc130b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:40.404 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.401+0000 7f6db9d9b640 1 -- 172.21.15.105:0/4152103686 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6dbc12c5b0 msgr2=0x7f6dbc130b60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:40.404 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.401+0000 7f6db9d9b640 1 --2- 172.21.15.105:0/4152103686 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6dbc12c5b0 0x7f6dbc130b60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:40.404 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.401+0000 7f6db9d9b640 1 -- 172.21.15.105:0/4152103686 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6dbc06eb30 msgr2=0x7f6dbc1300e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:40.404 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.401+0000 7f6db9d9b640 1 --2- 172.21.15.105:0/4152103686 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6dbc06eb30 0x7f6dbc1300e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:40.404 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.401+0000 7f6db9d9b640 1 -- 172.21.15.105:0/4152103686 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6dac009490 con 0x7f6dbc06f450 2024-09-21T01:44:40.404 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.401+0000 7f6dba59c640 1 --2- 172.21.15.105:0/4152103686 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6dbc06eb30 0x7f6dbc1300e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:40.404 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.401+0000 7f6db9d9b640 1 --2- 172.21.15.105:0/4152103686 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6dbc06f450 0x7f6dbc130620 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7f6db000e9c0 tx=0x7f6db000ee80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:40.404 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.402+0000 7f6daaffd640 1 -- 172.21.15.105:0/4152103686 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6db0009700 con 0x7f6dbc06f450 2024-09-21T01:44:40.405 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.402+0000 7f6daaffd640 1 -- 172.21.15.105:0/4152103686 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f6db000cd80 con 0x7f6dbc06f450 2024-09-21T01:44:40.405 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.402+0000 7f6dc0df4640 1 -- 172.21.15.105:0/4152103686 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6dbc132700 con 0x7f6dbc06f450 2024-09-21T01:44:40.405 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.402+0000 7f6daaffd640 1 -- 172.21.15.105:0/4152103686 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6db0004280 con 0x7f6dbc06f450 2024-09-21T01:44:40.405 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.402+0000 7f6dc0df4640 1 -- 172.21.15.105:0/4152103686 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6dbc134460 con 0x7f6dbc06f450 2024-09-21T01:44:40.405 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.402+0000 7f6daaffd640 1 -- 172.21.15.105:0/4152103686 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 2) v1 ==== 27607+0+0 (secure 0 0 0) 0x7f6db00035f0 con 0x7f6dbc06f450 2024-09-21T01:44:40.405 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.402+0000 7f6daaffd640 1 -- 172.21.15.105:0/4152103686 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(6..6 src has 1..6) v4 ==== 1185+0+0 (secure 0 0 0) 0x7f6db0014070 con 0x7f6dbc06f450 2024-09-21T01:44:40.405 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.403+0000 7f6dc0df4640 1 -- 172.21.15.105:0/4152103686 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6dbc133790 con 0x7f6dbc06f450 2024-09-21T01:44:40.405 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.405+0000 7f6daaffd640 1 -- 172.21.15.105:0/4152103686 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+73245 (secure 0 0 0) 0x7f6db000f9b0 con 0x7f6dbc06f450 2024-09-21T01:44:40.462 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.461+0000 7f6dc0df4640 1 -- 172.21.15.105:0/4152103686 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f6dbc134740 con 0x7f6dbc06f450 2024-09-21T01:44:40.462 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.462+0000 7f6daaffd640 1 -- 172.21.15.105:0/4152103686 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v6) v1 ==== 74+0+4152 (secure 0 0 0) 0x7f6db000f3c0 con 0x7f6dbc06f450 2024-09-21T01:44:40.462 INFO:teuthology.misc.health.smithi105.stdout: 2024-09-21T01:44:40.462 INFO:teuthology.misc.health.smithi105.stdout:{"epoch":6,"fsid":"355399d3-6490-4ad0-ac52-d793648ff276","created":"2024-09-21T01:44:37.282024+0000","modified":"2024-09-21T01:44:39.666350+0000","last_up_change":"0.000000","last_in_change":"2024-09-21T01:44:39.666350+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"03218bdb-ec48-44c7-bbf5-31cb53a3cea7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"5528ff44-922a-4e44-bb69-95fd9544d8dd","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"df56ff35-71e9-4d46-8abb-84265eb6e1ee","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"012d9668-f1d3-4268-b0eb-15fa845ef17c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T01:44:40.463 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.462+0000 7f6dc0df4640 1 -- 172.21.15.105:0/4152103686 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6dbc06f450 msgr2=0x7f6dbc130620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:40.463 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.462+0000 7f6dc0df4640 1 --2- 172.21.15.105:0/4152103686 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6dbc06f450 0x7f6dbc130620 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7f6db000e9c0 tx=0x7f6db000ee80 comp rx=0 tx=0).stop 2024-09-21T01:44:40.463 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.462+0000 7f6dc0df4640 1 -- 172.21.15.105:0/4152103686 shutdown_connections 2024-09-21T01:44:40.463 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.462+0000 7f6dc0df4640 1 --2- 172.21.15.105:0/4152103686 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6dbc12c5b0 0x7f6dbc130b60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:40.464 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.462+0000 7f6dc0df4640 1 --2- 172.21.15.105:0/4152103686 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6dbc06f450 0x7f6dbc130620 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:40.464 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.462+0000 7f6dc0df4640 1 --2- 172.21.15.105:0/4152103686 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6dbc06eb30 0x7f6dbc1300e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:40.464 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.462+0000 7f6dc0df4640 1 -- 172.21.15.105:0/4152103686 >> 172.21.15.105:0/4152103686 conn(0x7f6dbc09b5d0 msgr2=0x7f6dbc125ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:40.464 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.463+0000 7f6dc0df4640 1 -- 172.21.15.105:0/4152103686 shutdown_connections 2024-09-21T01:44:40.464 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:40.463+0000 7f6dc0df4640 1 -- 172.21.15.105:0/4152103686 wait complete. 2024-09-21T01:44:40.472 DEBUG:teuthology.misc:0 of 4 OSDs are up 2024-09-21T01:44:41.353 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:41.352+0000 7fc746329640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-21T01:44:41.354 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:44:41.354+0000 7fc746329640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-21T01:44:43.544 INFO:tasks.ceph.osd.0.smithi105.stderr:2024-09-21T01:44:43.544+0000 7fbf67a9c640 -1 osd.0 0 log_to_monitors true 2024-09-21T01:44:43.586 INFO:tasks.ceph.osd.2.smithi105.stderr:2024-09-21T01:44:43.586+0000 7f4efbf34640 -1 osd.2 0 log_to_monitors true 2024-09-21T01:44:43.790 INFO:tasks.ceph.osd.3.smithi105.stderr:2024-09-21T01:44:43.790+0000 7fe28a145640 -1 osd.3 0 log_to_monitors true 2024-09-21T01:44:43.858 INFO:tasks.ceph.osd.1.smithi105.stderr:2024-09-21T01:44:43.858+0000 7fcfba212640 -1 osd.1 0 log_to_monitors true 2024-09-21T01:44:46.875 DEBUG:teuthology.orchestra.run.smithi105:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-09-21T01:44:47.012 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.011+0000 7facd671f640 1 -- 172.21.15.105:0/2770016367 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7facd0136070 msgr2=0x7facd0138460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:47.012 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.011+0000 7facd671f640 1 --2- 172.21.15.105:0/2770016367 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7facd0136070 0x7facd0138460 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7facc8005fd0 tx=0x7facc802a930 comp rx=0 tx=0).stop 2024-09-21T01:44:47.012 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.011+0000 7facd671f640 1 -- 172.21.15.105:0/2770016367 shutdown_connections 2024-09-21T01:44:47.012 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.011+0000 7facd671f640 1 --2- 172.21.15.105:0/2770016367 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7facd0136070 0x7facd0138460 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:47.012 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.011+0000 7facd671f640 1 --2- 172.21.15.105:0/2770016367 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7facd0089000 0x7facd00893e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:47.013 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.011+0000 7facd671f640 1 -- 172.21.15.105:0/2770016367 >> 172.21.15.105:0/2770016367 conn(0x7facd00973a0 msgr2=0x7facd00977b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:47.013 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.011+0000 7facd671f640 1 -- 172.21.15.105:0/2770016367 shutdown_connections 2024-09-21T01:44:47.013 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.011+0000 7facd671f640 1 -- 172.21.15.105:0/2770016367 wait complete. 2024-09-21T01:44:47.013 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.013+0000 7facd671f640 1 Processor -- start 2024-09-21T01:44:47.016 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.013+0000 7facd671f640 1 -- start start 2024-09-21T01:44:47.016 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.014+0000 7facd671f640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7facd0089000 0x7facd011f780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:47.016 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.014+0000 7facd671f640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7facd0089920 0x7facd011fcc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:47.016 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.014+0000 7facd671f640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7facd0136070 0x7facd0120200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:47.016 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.014+0000 7facd671f640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7facd0121b40 con 0x7facd0136070 2024-09-21T01:44:47.017 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.014+0000 7facd671f640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7facd01266b0 con 0x7facd0089000 2024-09-21T01:44:47.017 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.014+0000 7facd671f640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7facd0122320 con 0x7facd0089920 2024-09-21T01:44:47.017 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.014+0000 7facc77fe640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7facd0089920 0x7facd011fcc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:47.017 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.014+0000 7facc77fe640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7facd0089920 0x7facd011fcc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:46984/0 (socket says 172.21.15.105:46984) 2024-09-21T01:44:47.017 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.014+0000 7facc77fe640 1 -- 172.21.15.105:0/345512643 learned_addr learned my addr 172.21.15.105:0/345512643 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:47.017 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.015+0000 7faccffff640 1 --2- 172.21.15.105:0/345512643 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7facd0089000 0x7facd011f780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:47.017 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.015+0000 7faccffff640 1 -- 172.21.15.105:0/345512643 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7facd0089920 msgr2=0x7facd011fcc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:47.017 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.015+0000 7faccffff640 1 --2- 172.21.15.105:0/345512643 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7facd0089920 0x7facd011fcc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:47.017 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.015+0000 7faccffff640 1 -- 172.21.15.105:0/345512643 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7facd0136070 msgr2=0x7facd0120200 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:44:47.017 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.015+0000 7facd4cbc640 1 --2- 172.21.15.105:0/345512643 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7facd0136070 0x7facd0120200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:47.017 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.015+0000 7faccffff640 1 --2- 172.21.15.105:0/345512643 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7facd0136070 0x7facd0120200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:47.017 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.015+0000 7faccffff640 1 -- 172.21.15.105:0/345512643 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7facc0009490 con 0x7facd0089000 2024-09-21T01:44:47.017 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.015+0000 7facc77fe640 1 --2- 172.21.15.105:0/345512643 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7facd0089920 0x7facd011fcc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:47.017 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.016+0000 7faccffff640 1 --2- 172.21.15.105:0/345512643 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7facd0089000 0x7facd011f780 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7facc000ea20 tx=0x7facc000eee0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:47.018 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.016+0000 7facd4cbc640 1 --2- 172.21.15.105:0/345512643 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7facd0136070 0x7facd0120200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T01:44:47.018 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.016+0000 7faccd7fa640 1 -- 172.21.15.105:0/345512643 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7facc000cd50 con 0x7facd0089000 2024-09-21T01:44:47.018 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.016+0000 7facd671f640 1 -- 172.21.15.105:0/345512643 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7facc8009900 con 0x7facd0089000 2024-09-21T01:44:47.019 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.016+0000 7facd671f640 1 -- 172.21.15.105:0/345512643 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7facd01212b0 con 0x7facd0089000 2024-09-21T01:44:47.019 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.016+0000 7faccd7fa640 1 -- 172.21.15.105:0/345512643 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7facc0004020 con 0x7facd0089000 2024-09-21T01:44:47.019 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.016+0000 7faccd7fa640 1 -- 172.21.15.105:0/345512643 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7facc0004410 con 0x7facd0089000 2024-09-21T01:44:47.019 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.018+0000 7facd671f640 1 -- 172.21.15.105:0/345512643 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7facb4005250 con 0x7facd0089000 2024-09-21T01:44:47.019 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.018+0000 7faccd7fa640 1 -- 172.21.15.105:0/345512643 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 4) v1 ==== 55156+0+0 (secure 0 0 0) 0x7facc0005020 con 0x7facd0089000 2024-09-21T01:44:47.019 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.019+0000 7faccd7fa640 1 -- 172.21.15.105:0/345512643 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(8..8 src has 1..8) v4 ==== 1513+0+0 (secure 0 0 0) 0x7facc0014070 con 0x7facd0089000 2024-09-21T01:44:47.027 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.026+0000 7faccd7fa640 1 -- 172.21.15.105:0/345512643 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7facc003aab0 con 0x7facd0089000 2024-09-21T01:44:47.274 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.273+0000 7facd671f640 1 -- 172.21.15.105:0/345512643 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7facb4005500 con 0x7facd0089000 2024-09-21T01:44:47.275 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.274+0000 7faccd7fa640 1 -- 172.21.15.105:0/345512643 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v8) v1 ==== 74+0+4152 (secure 0 0 0) 0x7facc0046090 con 0x7facd0089000 2024-09-21T01:44:47.275 INFO:teuthology.misc.health.smithi105.stdout: 2024-09-21T01:44:47.275 INFO:teuthology.misc.health.smithi105.stdout:{"epoch":8,"fsid":"355399d3-6490-4ad0-ac52-d793648ff276","created":"2024-09-21T01:44:37.282024+0000","modified":"2024-09-21T01:44:45.359093+0000","last_up_change":"0.000000","last_in_change":"2024-09-21T01:44:39.666350+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"03218bdb-ec48-44c7-bbf5-31cb53a3cea7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"5528ff44-922a-4e44-bb69-95fd9544d8dd","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"df56ff35-71e9-4d46-8abb-84265eb6e1ee","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":3,"uuid":"012d9668-f1d3-4268-b0eb-15fa845ef17c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T01:44:47.278 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.277+0000 7facd671f640 1 -- 172.21.15.105:0/345512643 >> v1:172.21.15.105:6816/3933313069 conn(0x7facb0047500 legacy=0x7facb00499c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:47.278 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.277+0000 7facd671f640 1 -- 172.21.15.105:0/345512643 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7facd0089000 msgr2=0x7facd011f780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:47.278 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.277+0000 7facd671f640 1 --2- 172.21.15.105:0/345512643 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7facd0089000 0x7facd011f780 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7facc000ea20 tx=0x7facc000eee0 comp rx=0 tx=0).stop 2024-09-21T01:44:47.278 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.277+0000 7facd671f640 1 -- 172.21.15.105:0/345512643 shutdown_connections 2024-09-21T01:44:47.278 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.277+0000 7facd671f640 1 --2- 172.21.15.105:0/345512643 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7facd0136070 0x7facd0120200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:47.278 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.277+0000 7facd671f640 1 --2- 172.21.15.105:0/345512643 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7facd0089920 0x7facd011fcc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:47.278 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.277+0000 7facd671f640 1 --2- 172.21.15.105:0/345512643 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7facd0089000 0x7facd011f780 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:47.278 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.277+0000 7facd671f640 1 -- 172.21.15.105:0/345512643 >> 172.21.15.105:0/345512643 conn(0x7facd00973a0 msgr2=0x7facd0126860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:47.278 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.277+0000 7facd671f640 1 -- 172.21.15.105:0/345512643 shutdown_connections 2024-09-21T01:44:47.278 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:47.277+0000 7facd671f640 1 -- 172.21.15.105:0/345512643 wait complete. 2024-09-21T01:44:47.289 DEBUG:teuthology.misc:0 of 4 OSDs are up 2024-09-21T01:44:47.469 INFO:tasks.ceph.osd.0.smithi105.stderr:2024-09-21T01:44:47.467+0000 7fbf63206640 -1 osd.0 0 waiting for initial osdmap 2024-09-21T01:44:47.480 INFO:tasks.ceph.osd.1.smithi105.stderr:2024-09-21T01:44:47.480+0000 7fcfb5279640 -1 osd.1 0 waiting for initial osdmap 2024-09-21T01:44:47.480 INFO:tasks.ceph.osd.0.smithi105.stderr:2024-09-21T01:44:47.480+0000 7fbf5b839640 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T01:44:47.484 INFO:tasks.ceph.osd.1.smithi105.stderr:2024-09-21T01:44:47.484+0000 7fcfad8ac640 -1 osd.1 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T01:44:47.489 INFO:tasks.ceph.osd.3.smithi105.stderr:2024-09-21T01:44:47.489+0000 7fe2858af640 -1 osd.3 0 waiting for initial osdmap 2024-09-21T01:44:47.494 INFO:tasks.ceph.osd.3.smithi105.stderr:2024-09-21T01:44:47.494+0000 7fe27dee2640 -1 osd.3 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T01:44:47.518 INFO:tasks.ceph.osd.2.smithi105.stderr:2024-09-21T01:44:47.518+0000 7f4ef769e640 -1 osd.2 0 waiting for initial osdmap 2024-09-21T01:44:47.521 INFO:tasks.ceph.osd.2.smithi105.stderr:2024-09-21T01:44:47.521+0000 7f4eefcd1640 -1 osd.2 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T01:44:53.692 DEBUG:teuthology.orchestra.run.smithi105:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-09-21T01:44:53.794 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.793+0000 7f5cd6e61640 1 -- 172.21.15.105:0/3487837515 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5cd006f2b0 msgr2=0x7f5cd006f710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:53.795 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.793+0000 7f5cd6e61640 1 --2- 172.21.15.105:0/3487837515 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5cd006f2b0 0x7f5cd006f710 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f5cbc009860 tx=0x7f5cbc02a3d0 comp rx=0 tx=0).stop 2024-09-21T01:44:53.795 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.793+0000 7f5cd6e61640 1 -- 172.21.15.105:0/3487837515 shutdown_connections 2024-09-21T01:44:53.795 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.793+0000 7f5cd6e61640 1 --2- 172.21.15.105:0/3487837515 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5cd006f2b0 0x7f5cd006f710 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:53.795 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.793+0000 7f5cd6e61640 1 --2- 172.21.15.105:0/3487837515 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f5cd012a3c0 0x7f5cd012a7a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:53.795 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.793+0000 7f5cd6e61640 1 -- 172.21.15.105:0/3487837515 >> 172.21.15.105:0/3487837515 conn(0x7f5cd009b0f0 msgr2=0x7f5cd009b500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:53.795 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.793+0000 7f5cd6e61640 1 -- 172.21.15.105:0/3487837515 shutdown_connections 2024-09-21T01:44:53.795 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.793+0000 7f5cd6e61640 1 -- 172.21.15.105:0/3487837515 wait complete. 2024-09-21T01:44:53.795 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.794+0000 7f5cd6e61640 1 Processor -- start 2024-09-21T01:44:53.795 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.794+0000 7f5cd6e61640 1 -- start start 2024-09-21T01:44:53.797 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.795+0000 7f5cd6e61640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f5cd006f2b0 0x7f5cd00896f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:53.797 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.795+0000 7f5cd6e61640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5cd006fc50 0x7f5cd0089c30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:53.797 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.795+0000 7f5cd6e61640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5cd012a3c0 0x7f5cd008a170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:53.797 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.795+0000 7f5cd6e61640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5cd008ab10 con 0x7f5cd006fc50 2024-09-21T01:44:53.797 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.795+0000 7f5cd6e61640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f5cd008c770 con 0x7f5cd006f2b0 2024-09-21T01:44:53.798 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.795+0000 7f5cd6e61640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f5cd0071a90 con 0x7f5cd012a3c0 2024-09-21T01:44:53.798 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.795+0000 7f5cd4bfd640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f5cd006f2b0 0x7f5cd00896f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:53.798 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.795+0000 7f5ccffff640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5cd006fc50 0x7f5cd0089c30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:53.798 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.795+0000 7f5cd4bfd640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f5cd006f2b0 0x7f5cd00896f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:58630/0 (socket says 172.21.15.105:58630) 2024-09-21T01:44:53.798 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.795+0000 7f5cd4bfd640 1 -- 172.21.15.105:0/4286939533 learned_addr learned my addr 172.21.15.105:0/4286939533 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:53.798 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.795+0000 7f5cd53fe640 1 --2- 172.21.15.105:0/4286939533 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5cd012a3c0 0x7f5cd008a170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:53.798 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.795+0000 7f5cd53fe640 1 -- 172.21.15.105:0/4286939533 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f5cd006f2b0 msgr2=0x7f5cd00896f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:53.798 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.795+0000 7f5cd53fe640 1 --2- 172.21.15.105:0/4286939533 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f5cd006f2b0 0x7f5cd00896f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:53.798 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.795+0000 7f5cd53fe640 1 -- 172.21.15.105:0/4286939533 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5cd006fc50 msgr2=0x7f5cd0089c30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:53.798 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.795+0000 7f5cd53fe640 1 --2- 172.21.15.105:0/4286939533 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5cd006fc50 0x7f5cd0089c30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:53.798 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.795+0000 7f5cd53fe640 1 -- 172.21.15.105:0/4286939533 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5cc4009490 con 0x7f5cd012a3c0 2024-09-21T01:44:53.798 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.795+0000 7f5ccffff640 1 --2- 172.21.15.105:0/4286939533 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5cd006fc50 0x7f5cd0089c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:53.798 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.795+0000 7f5cd4bfd640 1 --2- 172.21.15.105:0/4286939533 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f5cd006f2b0 0x7f5cd00896f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:53.798 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.796+0000 7f5cd53fe640 1 --2- 172.21.15.105:0/4286939533 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5cd012a3c0 0x7f5cd008a170 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f5cc800caa0 tx=0x7f5cc800cf60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:53.799 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.796+0000 7f5ccd7fa640 1 -- 172.21.15.105:0/4286939533 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5cc800ac00 con 0x7f5cd012a3c0 2024-09-21T01:44:53.799 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.796+0000 7f5ccd7fa640 1 -- 172.21.15.105:0/4286939533 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f5cc800ae50 con 0x7f5cd012a3c0 2024-09-21T01:44:53.799 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.796+0000 7f5cd6e61640 1 -- 172.21.15.105:0/4286939533 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5cbc009490 con 0x7f5cd012a3c0 2024-09-21T01:44:53.799 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.796+0000 7f5ccd7fa640 1 -- 172.21.15.105:0/4286939533 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f5cc8012680 con 0x7f5cd012a3c0 2024-09-21T01:44:53.799 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.796+0000 7f5cd6e61640 1 -- 172.21.15.105:0/4286939533 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5cd008b260 con 0x7f5cd012a3c0 2024-09-21T01:44:53.799 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.797+0000 7f5ccd7fa640 1 -- 172.21.15.105:0/4286939533 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f5cc801a3b0 con 0x7f5cd012a3c0 2024-09-21T01:44:53.799 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.797+0000 7f5cd6e61640 1 -- 172.21.15.105:0/4286939533 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5cd006f710 con 0x7f5cd012a3c0 2024-09-21T01:44:53.799 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.797+0000 7f5ccd7fa640 1 -- 172.21.15.105:0/4286939533 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(13..13 src has 1..13) v4 ==== 2459+0+0 (secure 0 0 0) 0x7f5cc805c970 con 0x7f5cd012a3c0 2024-09-21T01:44:53.801 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.801+0000 7f5ccd7fa640 1 -- 172.21.15.105:0/4286939533 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f5cc803a7c0 con 0x7f5cd012a3c0 2024-09-21T01:44:53.938 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.938+0000 7f5cd6e61640 1 -- 172.21.15.105:0/4286939533 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f5cd012aed0 con 0x7f5cd012a3c0 2024-09-21T01:44:53.938 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.938+0000 7f5ccd7fa640 1 -- 172.21.15.105:0/4286939533 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v13) v1 ==== 74+0+6989 (secure 0 0 0) 0x7f5cc8016070 con 0x7f5cd012a3c0 2024-09-21T01:44:53.939 INFO:teuthology.misc.health.smithi105.stdout: 2024-09-21T01:44:53.939 INFO:teuthology.misc.health.smithi105.stdout:{"epoch":13,"fsid":"355399d3-6490-4ad0-ac52-d793648ff276","created":"2024-09-21T01:44:37.282024+0000","modified":"2024-09-21T01:44:52.499119+0000","last_up_change":"2024-09-21T01:44:48.482537+0000","last_in_change":"2024-09-21T01:44:39.666350+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T01:44:50.380956+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"03218bdb-ec48-44c7-bbf5-31cb53a3cea7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6800","nonce":4053719786}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6801","nonce":4053719786}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6803","nonce":4053719786}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6802","nonce":4053719786}]},"public_addr":"172.21.15.105:6800/4053719786","cluster_addr":"172.21.15.105:6801/4053719786","heartbeat_back_addr":"172.21.15.105:6803/4053719786","heartbeat_front_addr":"172.21.15.105:6802/4053719786","state":["exists","up"]},{"osd":1,"uuid":"5528ff44-922a-4e44-bb69-95fd9544d8dd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6812","nonce":1313742696}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6813","nonce":1313742696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6815","nonce":1313742696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6814","nonce":1313742696}]},"public_addr":"172.21.15.105:6812/1313742696","cluster_addr":"172.21.15.105:6813/1313742696","heartbeat_back_addr":"172.21.15.105:6815/1313742696","heartbeat_front_addr":"172.21.15.105:6814/1313742696","state":["exists","up"]},{"osd":2,"uuid":"df56ff35-71e9-4d46-8abb-84265eb6e1ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6804","nonce":2159605016}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6805","nonce":2159605016}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6807","nonce":2159605016}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6806","nonce":2159605016}]},"public_addr":"172.21.15.105:6804/2159605016","cluster_addr":"172.21.15.105:6805/2159605016","heartbeat_back_addr":"172.21.15.105:6807/2159605016","heartbeat_front_addr":"172.21.15.105:6806/2159605016","state":["exists","up"]},{"osd":3,"uuid":"012d9668-f1d3-4268-b0eb-15fa845ef17c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6808","nonce":2514115267}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6809","nonce":2514115267}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6811","nonce":2514115267}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6810","nonce":2514115267}]},"public_addr":"172.21.15.105:6808/2514115267","cluster_addr":"172.21.15.105:6809/2514115267","heartbeat_back_addr":"172.21.15.105:6811/2514115267","heartbeat_front_addr":"172.21.15.105:6810/2514115267","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.554795+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.869400+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.601687+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.822680+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T01:44:53.940 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.940+0000 7f5cd6e61640 1 -- 172.21.15.105:0/4286939533 >> v1:172.21.15.105:6816/3933313069 conn(0x7f5cb4047560 legacy=0x7f5cb4049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:53.940 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.940+0000 7f5cd6e61640 1 -- 172.21.15.105:0/4286939533 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5cd012a3c0 msgr2=0x7f5cd008a170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:53.940 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.940+0000 7f5cd6e61640 1 --2- 172.21.15.105:0/4286939533 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5cd012a3c0 0x7f5cd008a170 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f5cc800caa0 tx=0x7f5cc800cf60 comp rx=0 tx=0).stop 2024-09-21T01:44:53.941 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.940+0000 7f5cd6e61640 1 -- 172.21.15.105:0/4286939533 shutdown_connections 2024-09-21T01:44:53.941 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.940+0000 7f5cd6e61640 1 --2- 172.21.15.105:0/4286939533 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5cd012a3c0 0x7f5cd008a170 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:53.941 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.940+0000 7f5cd6e61640 1 --2- 172.21.15.105:0/4286939533 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5cd006fc50 0x7f5cd0089c30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:53.941 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.940+0000 7f5cd6e61640 1 --2- 172.21.15.105:0/4286939533 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f5cd006f2b0 0x7f5cd00896f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:53.941 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.940+0000 7f5cd6e61640 1 -- 172.21.15.105:0/4286939533 >> 172.21.15.105:0/4286939533 conn(0x7f5cd009b0f0 msgr2=0x7f5cd0093c60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:53.941 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.940+0000 7f5cd6e61640 1 -- 172.21.15.105:0/4286939533 shutdown_connections 2024-09-21T01:44:53.941 INFO:teuthology.misc.health.smithi105.stderr:2024-09-21T01:44:53.940+0000 7f5cd6e61640 1 -- 172.21.15.105:0/4286939533 wait complete. 2024-09-21T01:44:53.950 DEBUG:teuthology.misc:4 of 4 OSDs are up 2024-09-21T01:44:53.950 INFO:tasks.ceph:Creating RBD pool 2024-09-21T01:44:53.950 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph osd pool create rbd 8 2024-09-21T01:44:54.049 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.048+0000 7f3569661640 1 -- 172.21.15.105:0/1465487767 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3564126700 msgr2=0x7f3564126ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:54.049 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.048+0000 7f3569661640 1 --2- 172.21.15.105:0/1465487767 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3564126700 0x7f3564126ae0 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f3554009830 tx=0x7f355402a3f0 comp rx=0 tx=0).stop 2024-09-21T01:44:54.049 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.048+0000 7f3569661640 1 -- 172.21.15.105:0/1465487767 shutdown_connections 2024-09-21T01:44:54.049 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.048+0000 7f3569661640 1 --2- 172.21.15.105:0/1465487767 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3564127020 0x7f3564129c60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:54.049 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.048+0000 7f3569661640 1 --2- 172.21.15.105:0/1465487767 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3564126700 0x7f3564126ae0 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:54.049 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.048+0000 7f3569661640 1 -- 172.21.15.105:0/1465487767 >> 172.21.15.105:0/1465487767 conn(0x7f35640995c0 msgr2=0x7f35640999d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:54.049 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.048+0000 7f3569661640 1 -- 172.21.15.105:0/1465487767 shutdown_connections 2024-09-21T01:44:54.049 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.048+0000 7f3569661640 1 -- 172.21.15.105:0/1465487767 wait complete. 2024-09-21T01:44:54.051 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.049+0000 7f3569661640 1 Processor -- start 2024-09-21T01:44:54.051 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.049+0000 7f3569661640 1 -- start start 2024-09-21T01:44:54.051 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.050+0000 7f3569661640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3564126700 0x7f35641c20a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:54.051 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.050+0000 7f3569661640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3564127020 0x7f35641c25e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:54.051 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.050+0000 7f3569661640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f356412a1a0 0x7f356406ba10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:54.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.050+0000 7f3569661640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3564113180 con 0x7f3564127020 2024-09-21T01:44:54.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.050+0000 7f3569661640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3564113f40 con 0x7f356412a1a0 2024-09-21T01:44:54.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.050+0000 7f3569661640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f356406d020 con 0x7f3564126700 2024-09-21T01:44:54.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.050+0000 7f3562ffd640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3564126700 0x7f35641c20a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:54.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.050+0000 7f3562ffd640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3564126700 0x7f35641c20a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:55238/0 (socket says 172.21.15.105:55238) 2024-09-21T01:44:54.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.050+0000 7f3562ffd640 1 -- 172.21.15.105:0/2244670681 learned_addr learned my addr 172.21.15.105:0/2244670681 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:54.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.050+0000 7f35637fe640 1 --2- 172.21.15.105:0/2244670681 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f356412a1a0 0x7f356406ba10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:54.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.050+0000 7f355a7fc640 1 --2- 172.21.15.105:0/2244670681 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3564127020 0x7f35641c25e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:54.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.050+0000 7f3562ffd640 1 -- 172.21.15.105:0/2244670681 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f356412a1a0 msgr2=0x7f356406ba10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:54.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.050+0000 7f3562ffd640 1 --2- 172.21.15.105:0/2244670681 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f356412a1a0 0x7f356406ba10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:54.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.050+0000 7f3562ffd640 1 -- 172.21.15.105:0/2244670681 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3564127020 msgr2=0x7f35641c25e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:54.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.050+0000 7f3562ffd640 1 --2- 172.21.15.105:0/2244670681 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3564127020 0x7f35641c25e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:54.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.050+0000 7f3562ffd640 1 -- 172.21.15.105:0/2244670681 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f354c009560 con 0x7f3564126700 2024-09-21T01:44:54.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.050+0000 7f355a7fc640 1 --2- 172.21.15.105:0/2244670681 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3564127020 0x7f35641c25e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:54.053 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.050+0000 7f3562ffd640 1 --2- 172.21.15.105:0/2244670681 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3564126700 0x7f35641c20a0 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f3554000c00 tx=0x7f3554003c10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:54.053 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.050+0000 7f355bfff640 1 -- 172.21.15.105:0/2244670681 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3554005220 con 0x7f3564126700 2024-09-21T01:44:54.053 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.050+0000 7f355bfff640 1 -- 172.21.15.105:0/2244670681 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f3554005470 con 0x7f3564126700 2024-09-21T01:44:54.053 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.050+0000 7f3569661640 1 -- 172.21.15.105:0/2244670681 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3554009490 con 0x7f3564126700 2024-09-21T01:44:54.053 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.050+0000 7f355bfff640 1 -- 172.21.15.105:0/2244670681 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3554004a00 con 0x7f3564126700 2024-09-21T01:44:54.053 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.051+0000 7f3569661640 1 -- 172.21.15.105:0/2244670681 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f35641c2b20 con 0x7f3564126700 2024-09-21T01:44:54.053 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.052+0000 7f355bfff640 1 -- 172.21.15.105:0/2244670681 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f3554004b60 con 0x7f3564126700 2024-09-21T01:44:54.053 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.052+0000 7f3569661640 1 -- 172.21.15.105:0/2244670681 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f35641c2cb0 con 0x7f3564126700 2024-09-21T01:44:54.054 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.052+0000 7f355bfff640 1 -- 172.21.15.105:0/2244670681 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(13..13 src has 1..13) v4 ==== 2459+0+0 (secure 0 0 0) 0x7f355407d790 con 0x7f3564126700 2024-09-21T01:44:54.055 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.055+0000 7f355bfff640 1 -- 172.21.15.105:0/2244670681 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f355405f070 con 0x7f3564126700 2024-09-21T01:44:54.192 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.192+0000 7f3569661640 1 -- 172.21.15.105:0/2244670681 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd pool create", "pool": "rbd", "pg_num": 8} v 0) v1 -- 0x7f35641132f0 con 0x7f3564126700 2024-09-21T01:44:54.517 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.516+0000 7f355bfff640 1 -- 172.21.15.105:0/2244670681 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]=0 pool 'rbd' created v14) v1 ==== 109+0+0 (secure 0 0 0) 0x7f355405ec60 con 0x7f3564126700 2024-09-21T01:44:54.517 INFO:teuthology.orchestra.run.smithi105.stderr:pool 'rbd' created 2024-09-21T01:44:54.520 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.519+0000 7f3569661640 1 -- 172.21.15.105:0/2244670681 >> v1:172.21.15.105:6816/3933313069 conn(0x7f3544047560 legacy=0x7f3544049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:54.520 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.519+0000 7f3569661640 1 -- 172.21.15.105:0/2244670681 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3564126700 msgr2=0x7f35641c20a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:54.521 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.519+0000 7f3569661640 1 --2- 172.21.15.105:0/2244670681 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3564126700 0x7f35641c20a0 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f3554000c00 tx=0x7f3554003c10 comp rx=0 tx=0).stop 2024-09-21T01:44:54.521 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.519+0000 7f3569661640 1 -- 172.21.15.105:0/2244670681 shutdown_connections 2024-09-21T01:44:54.521 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.519+0000 7f3569661640 1 --2- 172.21.15.105:0/2244670681 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f356412a1a0 0x7f356406ba10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:54.521 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.519+0000 7f3569661640 1 --2- 172.21.15.105:0/2244670681 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3564127020 0x7f35641c25e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:54.521 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.519+0000 7f3569661640 1 --2- 172.21.15.105:0/2244670681 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3564126700 0x7f35641c20a0 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:54.521 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.519+0000 7f3569661640 1 -- 172.21.15.105:0/2244670681 >> 172.21.15.105:0/2244670681 conn(0x7f35640995c0 msgr2=0x7f3564124600 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:54.521 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.520+0000 7f3569661640 1 -- 172.21.15.105:0/2244670681 shutdown_connections 2024-09-21T01:44:54.521 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.520+0000 7f3569661640 1 -- 172.21.15.105:0/2244670681 wait complete. 2024-09-21T01:44:54.530 DEBUG:teuthology.orchestra.run.smithi105:> rbd --cluster ceph pool init rbd 2024-09-21T01:44:54.649 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.649+0000 7ff61b605c00 1 -- 172.21.15.105:0/3907787463 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x558251dd4d00 msgr2=0x558251dd70f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:54.649 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.649+0000 7ff61b605c00 1 --2- 172.21.15.105:0/3907787463 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x558251dd4d00 0x558251dd70f0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7ff61400a630 tx=0x7ff614046190 comp rx=0 tx=0).stop 2024-09-21T01:44:54.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.649+0000 7ff61b605c00 1 -- 172.21.15.105:0/3907787463 shutdown_connections 2024-09-21T01:44:54.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.649+0000 7ff61b605c00 1 --2- 172.21.15.105:0/3907787463 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x558251dd4d00 0x558251dd70f0 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:54.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.649+0000 7ff61b605c00 1 --2- 172.21.15.105:0/3907787463 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x558251dd23d0 0x558251dd47c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:54.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.649+0000 7ff61b605c00 1 -- 172.21.15.105:0/3907787463 >> 172.21.15.105:0/3907787463 conn(0x558251d3e950 msgr2=0x558251d3ed60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:54.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.649+0000 7ff61b605c00 1 -- 172.21.15.105:0/3907787463 shutdown_connections 2024-09-21T01:44:54.650 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.649+0000 7ff61b605c00 1 -- 172.21.15.105:0/3907787463 wait complete. 2024-09-21T01:44:54.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.649+0000 7ff61b605c00 1 Processor -- start 2024-09-21T01:44:54.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.650+0000 7ff61b605c00 1 -- start start 2024-09-21T01:44:54.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.650+0000 7ff61b605c00 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558251dcbe60 0x558251d38200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:54.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.650+0000 7ff61b605c00 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x558251dd23d0 0x558251d38740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:54.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.650+0000 7ff61b605c00 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x558251dd4d00 0x558251dc7800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:54.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.650+0000 7ff61b605c00 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x558251c17500 con 0x558251dcbe60 2024-09-21T01:44:54.653 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.650+0000 7ff61b605c00 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x558251bb2b90 con 0x558251dd23d0 2024-09-21T01:44:54.653 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.650+0000 7ff61b605c00 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x558251bb2830 con 0x558251dd4d00 2024-09-21T01:44:54.653 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.650+0000 7ff61b366640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x558251dd4d00 0x558251dc7800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:54.653 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.650+0000 7ff611903640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x558251dd23d0 0x558251d38740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:54.653 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.650+0000 7ff61a104640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558251dcbe60 0x558251d38200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:54.653 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.650+0000 7ff61b366640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x558251dd4d00 0x558251dc7800 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:55258/0 (socket says 172.21.15.105:55258) 2024-09-21T01:44:54.653 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.650+0000 7ff61b366640 1 -- 172.21.15.105:0/4209371465 learned_addr learned my addr 172.21.15.105:0/4209371465 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:54.653 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.650+0000 7ff611903640 1 -- 172.21.15.105:0/4209371465 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x558251dd4d00 msgr2=0x558251dc7800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:54.653 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.650+0000 7ff611903640 1 --2- 172.21.15.105:0/4209371465 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x558251dd4d00 0x558251dc7800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:54.653 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.650+0000 7ff611903640 1 -- 172.21.15.105:0/4209371465 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558251dcbe60 msgr2=0x558251d38200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:54.653 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.650+0000 7ff611903640 1 --2- 172.21.15.105:0/4209371465 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x558251dcbe60 0x558251d38200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:54.653 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.650+0000 7ff611903640 1 -- 172.21.15.105:0/4209371465 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff604004ba0 con 0x558251dd23d0 2024-09-21T01:44:54.653 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.650+0000 7ff61b366640 1 --2- 172.21.15.105:0/4209371465 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x558251dd4d00 0x558251dc7800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:54.653 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.650+0000 7ff611903640 1 --2- 172.21.15.105:0/4209371465 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x558251dd23d0 0x558251d38740 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7ff60400a680 tx=0x7ff60400ab40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:54.654 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.651+0000 7ff612ffd640 1 -- 172.21.15.105:0/4209371465 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff604010070 con 0x558251dd23d0 2024-09-21T01:44:54.654 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.651+0000 7ff612ffd640 1 -- 172.21.15.105:0/4209371465 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7ff604005040 con 0x558251dd23d0 2024-09-21T01:44:54.654 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.651+0000 7ff61b605c00 1 -- 172.21.15.105:0/4209371465 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff61400a260 con 0x558251dd23d0 2024-09-21T01:44:54.654 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.651+0000 7ff612ffd640 1 -- 172.21.15.105:0/4209371465 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff604008bf0 con 0x558251dd23d0 2024-09-21T01:44:54.654 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.651+0000 7ff61b605c00 1 -- 172.21.15.105:0/4209371465 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x558251bc1c50 con 0x558251dd23d0 2024-09-21T01:44:54.654 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.652+0000 7ff612ffd640 1 -- 172.21.15.105:0/4209371465 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7ff604008d50 con 0x558251dd23d0 2024-09-21T01:44:54.654 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.652+0000 7ff612ffd640 1 -- 172.21.15.105:0/4209371465 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(14..14 src has 1..14) v4 ==== 2801+0+0 (secure 0 0 0) 0x7ff60405af80 con 0x558251dd23d0 2024-09-21T01:44:54.654 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:54.652+0000 7ff61b605c00 1 -- 172.21.15.105:0/4209371465 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "osd pool application enable","pool": "rbd","app": "rbd"} v 0) v1 -- 0x558251bb5090 con 0x558251dd23d0 2024-09-21T01:44:55.515 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:55.515+0000 7ff612ffd640 1 -- 172.21.15.105:0/4209371465 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]=0 enabled application 'rbd' on pool 'rbd' v15) v1 ==== 141+0+0 (secure 0 0 0) 0x7ff604019070 con 0x558251dd23d0 2024-09-21T01:44:55.515 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:55.515+0000 7ff61b605c00 1 -- 172.21.15.105:0/4209371465 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x558251bdc990 con 0x558251dd23d0 2024-09-21T01:44:55.527 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:55.526+0000 7ff612ffd640 1 -- 172.21.15.105:0/4209371465 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_get_version_reply(handle=1 version=15) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff60403c700 con 0x558251dd23d0 2024-09-21T01:44:55.527 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:55.526+0000 7ff619102640 1 -- 172.21.15.105:0/4209371465 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=15}) v3 -- 0x7ff600002b60 con 0x558251dd23d0 2024-09-21T01:44:55.527 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:55.527+0000 7ff612ffd640 1 -- 172.21.15.105:0/4209371465 <== mon.1 v2:172.21.15.105:3301/0 8 ==== osd_map(15..15 src has 1..15) v4 ==== 650+0+0 (secure 0 0 0) 0x7ff60405a110 con 0x558251dd23d0 2024-09-21T01:44:55.527 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:55.527+0000 7ff61b605c00 1 -- 172.21.15.105:0/4209371465 --> v1:172.21.15.105:6808/2514115267 -- osd_op(unknown.0.0:1 2.3 2:c4c92e5a:::rbd_trash:head [create] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e15) v8 -- 0x558251de8620 con 0x558251dc8740 2024-09-21T01:44:55.533 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:55.533+0000 7ff61b366640 1 -- 172.21.15.105:0/4209371465 <== osd.3 v1:172.21.15.105:6808/2514115267 1 ==== osd_op_reply(1 rbd_trash [create] v15'1 uv1 ondisk = 0) v8 ==== 153+0+0 (unknown 2722885549 0 0) 0x7ff614052070 con 0x558251dc8740 2024-09-21T01:44:55.535 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:55.535+0000 7ff61b605c00 1 -- 172.21.15.105:0/4209371465 --> v1:172.21.15.105:6808/2514115267 -- osd_op(unknown.0.0:2 2.2 2:5cea7035:::rbd_info:head [call rbd.metadata_list in=33b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e15) v8 -- 0x558251e38230 con 0x558251dc8740 2024-09-21T01:44:55.536 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:55.535+0000 7ff61b366640 1 -- 172.21.15.105:0/4209371465 <== osd.3 v1:172.21.15.105:6808/2514115267 2 ==== osd_op_reply(2 rbd_info [call] v0'0 uv0 ondisk = -2 ((2) No such file or directory)) v8 ==== 152+0+0 (unknown 2354864950 0 0) 0x7ff614052070 con 0x558251dc8740 2024-09-21T01:44:55.536 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:55.535+0000 7ff61b605c00 1 -- 172.21.15.105:0/4209371465 --> v1:172.21.15.105:6808/2514115267 -- osd_op(unknown.0.0:3 2.2 2:5cea7035:::rbd_info:head [read 0~0] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e15) v8 -- 0x558251e37320 con 0x558251dc8740 2024-09-21T01:44:55.536 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:55.536+0000 7ff61b366640 1 -- 172.21.15.105:0/4209371465 <== osd.3 v1:172.21.15.105:6808/2514115267 3 ==== osd_op_reply(3 rbd_info [read 0~0] v0'0 uv0 ondisk = -2 ((2) No such file or directory)) v8 ==== 152+0+0 (unknown 2366413114 0 0) 0x7ff614052070 con 0x558251dc8740 2024-09-21T01:44:55.536 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:55.536+0000 7ff619102640 1 -- 172.21.15.105:0/4209371465 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- pool_op(create unmanaged snap pool 2 tid 4 name v0) v4 -- 0x7ff600003f30 con 0x558251dd23d0 2024-09-21T01:44:56.532 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:56.532+0000 7ff612ffd640 1 -- 172.21.15.105:0/4209371465 <== mon.1 v2:172.21.15.105:3301/0 9 ==== pool_op_reply(tid 4 (0) Success v16) v1 ==== 55+0+0 (secure 0 0 0) 0x7ff60403c2f0 con 0x558251dd23d0 2024-09-21T01:44:56.532 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:56.532+0000 7ff612ffd640 1 -- 172.21.15.105:0/4209371465 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=16}) v3 -- 0x7ff5fc04c940 con 0x558251dd23d0 2024-09-21T01:44:56.533 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:56.532+0000 7ff612ffd640 1 -- 172.21.15.105:0/4209371465 <== mon.1 v2:172.21.15.105:3301/0 10 ==== osd_map(16..16 src has 1..16) v4 ==== 626+0+0 (secure 0 0 0) 0x7ff60403aad0 con 0x558251dd23d0 2024-09-21T01:44:56.533 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:56.532+0000 7ff619102640 1 -- 172.21.15.105:0/4209371465 --> v1:172.21.15.105:6808/2514115267 -- osd_op(unknown.0.0:5 2.2 2:5cea7035:::rbd_info:head [create,write 0~8 in=8b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e16) v8 -- 0x7ff600005830 con 0x558251dc8740 2024-09-21T01:44:56.535 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:56.535+0000 7ff61b366640 1 -- 172.21.15.105:0/4209371465 <== osd.3 v1:172.21.15.105:6808/2514115267 4 ==== osd_op_reply(5 rbd_info [create,write 0~8] v16'1 uv1 ondisk = 0) v8 ==== 194+0+0 (unknown 1230306805 0 0) 0x7ff614052070 con 0x558251dc8740 2024-09-21T01:44:56.535 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:56.535+0000 7ff619102640 1 -- 172.21.15.105:0/4209371465 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- pool_op(delete unmanaged snap pool 2 tid 6 name v16) v4 -- 0x7ff600005340 con 0x558251dd23d0 2024-09-21T01:44:57.531 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.530+0000 7ff612ffd640 1 -- 172.21.15.105:0/4209371465 <== mon.1 v2:172.21.15.105:3301/0 11 ==== pool_op_reply(tid 6 (0) Success v17) v1 ==== 43+0+0 (secure 0 0 0) 0x7ff60403c4d0 con 0x558251dd23d0 2024-09-21T01:44:57.531 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.530+0000 7ff612ffd640 1 -- 172.21.15.105:0/4209371465 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=17}) v3 -- 0x7ff5fc04cad0 con 0x558251dd23d0 2024-09-21T01:44:57.531 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.531+0000 7ff612ffd640 1 -- 172.21.15.105:0/4209371465 <== mon.1 v2:172.21.15.105:3301/0 12 ==== osd_map(17..17 src has 1..17) v4 ==== 654+0+0 (secure 0 0 0) 0x7ff60403ad30 con 0x558251dd23d0 2024-09-21T01:44:57.531 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.531+0000 7ff619102640 1 -- 172.21.15.105:0/4209371465 --> v1:172.21.15.105:6808/2514115267 -- osd_op(unknown.0.0:7 2.2 2:5cea7035:::rbd_info:head [write 0~19 in=19b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e17) v8 -- 0x7ff6000060d0 con 0x558251dc8740 2024-09-21T01:44:57.536 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.535+0000 7ff61b366640 1 -- 172.21.15.105:0/4209371465 <== osd.3 v1:172.21.15.105:6808/2514115267 5 ==== osd_op_reply(7 rbd_info [write 0~19] v17'2 uv2 ondisk = 0) v8 ==== 152+0+0 (unknown 3144429724 0 0) 0x7ff614052070 con 0x558251dc8740 2024-09-21T01:44:57.536 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.535+0000 7ff61b605c00 1 -- 172.21.15.105:0/4209371465 >> v1:172.21.15.105:6808/2514115267 conn(0x558251dc8740 legacy=0x558251de6f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:57.536 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.535+0000 7ff61b605c00 1 -- 172.21.15.105:0/4209371465 >> v1:172.21.15.105:6816/3933313069 conn(0x7ff5fc045e50 legacy=0x7ff5fc048310 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:57.536 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.535+0000 7ff61b605c00 1 -- 172.21.15.105:0/4209371465 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x558251dd23d0 msgr2=0x558251d38740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:57.536 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.535+0000 7ff61b605c00 1 --2- 172.21.15.105:0/4209371465 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x558251dd23d0 0x558251d38740 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7ff60400a680 tx=0x7ff60400ab40 comp rx=0 tx=0).stop 2024-09-21T01:44:57.536 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.535+0000 7ff61b366640 1 -- 172.21.15.105:0/4209371465 reap_dead start 2024-09-21T01:44:57.536 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.535+0000 7ff61b605c00 1 -- 172.21.15.105:0/4209371465 shutdown_connections 2024-09-21T01:44:57.536 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.535+0000 7ff61b605c00 1 -- 172.21.15.105:0/4209371465 >> 172.21.15.105:0/4209371465 conn(0x558251d3e950 msgr2=0x558251d26cf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:57.536 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.535+0000 7ff61b605c00 1 -- 172.21.15.105:0/4209371465 shutdown_connections 2024-09-21T01:44:57.536 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.535+0000 7ff61b605c00 1 -- 172.21.15.105:0/4209371465 wait complete. 2024-09-21T01:44:57.540 INFO:tasks.ceph:Starting mds daemons in cluster ceph... 2024-09-21T01:44:57.540 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph config log 1 --format=json 2024-09-21T01:44:57.541 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2024-09-21T01:44:57.648 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.647+0000 7fa20ad9c640 1 -- 172.21.15.105:0/3247458833 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa204128fc0 msgr2=0x7fa2041341d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:57.648 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.647+0000 7fa20ad9c640 1 --2- 172.21.15.105:0/3247458833 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa204128fc0 0x7fa2041341d0 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7fa1fc005fd0 tx=0x7fa1fc02a780 comp rx=0 tx=0).stop 2024-09-21T01:44:57.648 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.647+0000 7fa20ad9c640 1 -- 172.21.15.105:0/3247458833 shutdown_connections 2024-09-21T01:44:57.648 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.647+0000 7fa20ad9c640 1 --2- 172.21.15.105:0/3247458833 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa204128fc0 0x7fa2041341d0 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:57.648 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.647+0000 7fa20ad9c640 1 --2- 172.21.15.105:0/3247458833 >> v2:172.21.15.105:3300/0 conn(0x7fa20412c430 0x7fa20412c810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:57.648 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.647+0000 7fa20ad9c640 1 -- 172.21.15.105:0/3247458833 >> 172.21.15.105:0/3247458833 conn(0x7fa20409b7c0 msgr2=0x7fa20409bbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:57.648 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.647+0000 7fa20ad9c640 1 -- 172.21.15.105:0/3247458833 shutdown_connections 2024-09-21T01:44:57.648 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.647+0000 7fa20ad9c640 1 -- 172.21.15.105:0/3247458833 wait complete. 2024-09-21T01:44:57.648 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.647+0000 7fa20ad9c640 1 Processor -- start 2024-09-21T01:44:57.648 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.648+0000 7fa20ad9c640 1 -- start start 2024-09-21T01:44:57.651 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.648+0000 7fa20ad9c640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa20409daa0 0x7fa20412fc90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:57.651 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.648+0000 7fa20ad9c640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa204128fc0 0x7fa2041301d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:57.651 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.648+0000 7fa20ad9c640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa20412c430 0x7fa204130710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:57.651 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.648+0000 7fa20ad9c640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa204131db0 con 0x7fa20412c430 2024-09-21T01:44:57.651 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.648+0000 7fa20ad9c640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa204132600 con 0x7fa204128fc0 2024-09-21T01:44:57.651 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.648+0000 7fa20ad9c640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fa2041331b0 con 0x7fa20409daa0 2024-09-21T01:44:57.651 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.648+0000 7fa208b38640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa20409daa0 0x7fa20412fc90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:57.651 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.648+0000 7fa203fff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa204128fc0 0x7fa2041301d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:57.651 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.648+0000 7fa208b38640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa20409daa0 0x7fa20412fc90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:55264/0 (socket says 172.21.15.105:55264) 2024-09-21T01:44:57.651 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.648+0000 7fa208b38640 1 -- 172.21.15.105:0/3154342708 learned_addr learned my addr 172.21.15.105:0/3154342708 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:57.651 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.648+0000 7fa209339640 1 --2- 172.21.15.105:0/3154342708 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa20412c430 0x7fa204130710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:57.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.648+0000 7fa208b38640 1 -- 172.21.15.105:0/3154342708 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa204128fc0 msgr2=0x7fa2041301d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:57.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.648+0000 7fa208b38640 1 --2- 172.21.15.105:0/3154342708 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa204128fc0 0x7fa2041301d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:57.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.649+0000 7fa208b38640 1 -- 172.21.15.105:0/3154342708 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa20412c430 msgr2=0x7fa204130710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:57.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.649+0000 7fa208b38640 1 --2- 172.21.15.105:0/3154342708 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa20412c430 0x7fa204130710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:57.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.649+0000 7fa208b38640 1 -- 172.21.15.105:0/3154342708 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa1f8009490 con 0x7fa20409daa0 2024-09-21T01:44:57.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.649+0000 7fa203fff640 1 --2- 172.21.15.105:0/3154342708 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa204128fc0 0x7fa2041301d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:57.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.649+0000 7fa208b38640 1 --2- 172.21.15.105:0/3154342708 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa20409daa0 0x7fa20412fc90 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7fa1f800ea60 tx=0x7fa1f800ef20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:57.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.649+0000 7fa2017fa640 1 -- 172.21.15.105:0/3154342708 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa1f800cde0 con 0x7fa20409daa0 2024-09-21T01:44:57.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.649+0000 7fa2017fa640 1 -- 172.21.15.105:0/3154342708 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fa1f8004020 con 0x7fa20409daa0 2024-09-21T01:44:57.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.649+0000 7fa20ad9c640 1 -- 172.21.15.105:0/3154342708 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa1fc009840 con 0x7fa20409daa0 2024-09-21T01:44:57.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.649+0000 7fa2017fa640 1 -- 172.21.15.105:0/3154342708 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa1f8004410 con 0x7fa20409daa0 2024-09-21T01:44:57.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.649+0000 7fa20ad9c640 1 -- 172.21.15.105:0/3154342708 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa2041314b0 con 0x7fa20409daa0 2024-09-21T01:44:57.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.650+0000 7fa20ad9c640 1 -- 172.21.15.105:0/3154342708 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa20412c810 con 0x7fa20409daa0 2024-09-21T01:44:57.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.650+0000 7fa2017fa640 1 -- 172.21.15.105:0/3154342708 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7fa1f80035f0 con 0x7fa20409daa0 2024-09-21T01:44:57.653 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.650+0000 7fa2017fa640 1 -- 172.21.15.105:0/3154342708 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7fa1f8014070 con 0x7fa20409daa0 2024-09-21T01:44:57.654 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.654+0000 7fa2017fa640 1 -- 172.21.15.105:0/3154342708 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7fa1f803af50 con 0x7fa20409daa0 2024-09-21T01:44:57.790 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.790+0000 7fa20ad9c640 1 -- 172.21.15.105:0/3154342708 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "config log", "num": 1, "format": "json"} v 0) v1 -- 0x7fa204124950 con 0x7fa20409daa0 2024-09-21T01:44:57.790 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.790+0000 7fa2017fa640 1 -- 172.21.15.105:0/3154342708 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "config log", "num": 1, "format": "json"}]=0 v27) v1 ==== 86+0+160 (secure 0 0 0) 0x7fa1f803e570 con 0x7fa20409daa0 2024-09-21T01:44:57.790 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:57.792 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.791+0000 7fa20ad9c640 1 -- 172.21.15.105:0/3154342708 >> v1:172.21.15.105:6816/3933313069 conn(0x7fa1e8047590 legacy=0x7fa1e8049a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:57.792 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.792+0000 7fa20ad9c640 1 -- 172.21.15.105:0/3154342708 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa20409daa0 msgr2=0x7fa20412fc90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:57.792 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.792+0000 7fa20ad9c640 1 --2- 172.21.15.105:0/3154342708 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa20409daa0 0x7fa20412fc90 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7fa1f800ea60 tx=0x7fa1f800ef20 comp rx=0 tx=0).stop 2024-09-21T01:44:57.792 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.792+0000 7fa20ad9c640 1 -- 172.21.15.105:0/3154342708 shutdown_connections 2024-09-21T01:44:57.792 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.792+0000 7fa20ad9c640 1 --2- 172.21.15.105:0/3154342708 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa20412c430 0x7fa204130710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:57.792 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.792+0000 7fa20ad9c640 1 --2- 172.21.15.105:0/3154342708 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa204128fc0 0x7fa2041301d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:57.793 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.792+0000 7fa20ad9c640 1 --2- 172.21.15.105:0/3154342708 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa20409daa0 0x7fa20412fc90 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:57.793 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.792+0000 7fa20ad9c640 1 -- 172.21.15.105:0/3154342708 >> 172.21.15.105:0/3154342708 conn(0x7fa20409b7c0 msgr2=0x7fa2041287a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:57.793 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.792+0000 7fa20ad9c640 1 -- 172.21.15.105:0/3154342708 shutdown_connections 2024-09-21T01:44:57.793 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.792+0000 7fa20ad9c640 1 -- 172.21.15.105:0/3154342708 wait complete. 2024-09-21T01:44:57.802 INFO:teuthology.orchestra.run.smithi105.stdout:[{"version":27,"timestamp":"2024-09-21T01:44:47.519649+0000","name":"","changes":[{"name":"osd.2/osd_mclock_max_capacity_iops_ssd","new_value":"5392.790338"}]}] 2024-09-21T01:44:57.803 INFO:tasks.ceph_manager:config epoch is 27 2024-09-21T01:44:57.803 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T01:44:57.803 INFO:tasks.ceph.ceph_manager.ceph:waiting for mgr available 2024-09-21T01:44:57.803 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mgr dump --format=json 2024-09-21T01:44:57.902 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.901+0000 7f6c49125640 1 -- 172.21.15.105:0/4081660346 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6c44128d40 msgr2=0x7f6c441291a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:57.902 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.901+0000 7f6c49125640 1 --2- 172.21.15.105:0/4081660346 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6c44128d40 0x7f6c441291a0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f6c38009890 tx=0x7f6c3802a4a0 comp rx=0 tx=0).stop 2024-09-21T01:44:57.902 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.901+0000 7f6c49125640 1 -- 172.21.15.105:0/4081660346 shutdown_connections 2024-09-21T01:44:57.902 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.901+0000 7f6c49125640 1 --2- 172.21.15.105:0/4081660346 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6c44128d40 0x7f6c441291a0 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:57.902 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.901+0000 7f6c49125640 1 --2- 172.21.15.105:0/4081660346 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6c4412c900 0x7f6c4412cce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:57.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.901+0000 7f6c49125640 1 -- 172.21.15.105:0/4081660346 >> 172.21.15.105:0/4081660346 conn(0x7f6c4409b7c0 msgr2=0x7f6c4409bbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:57.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.901+0000 7f6c49125640 1 -- 172.21.15.105:0/4081660346 shutdown_connections 2024-09-21T01:44:57.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.901+0000 7f6c49125640 1 -- 172.21.15.105:0/4081660346 wait complete. 2024-09-21T01:44:57.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.903+0000 7f6c49125640 1 Processor -- start 2024-09-21T01:44:57.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.903+0000 7f6c49125640 1 -- start start 2024-09-21T01:44:57.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.903+0000 7f6c49125640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6c44128d40 0x7f6c441c1930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:57.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.903+0000 7f6c49125640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6c441296e0 0x7f6c441c1e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:57.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.903+0000 7f6c49125640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6c4412c900 0x7f6c441c23b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:57.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.903+0000 7f6c49125640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6c44132040 con 0x7f6c441296e0 2024-09-21T01:44:57.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.903+0000 7f6c49125640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f6c441316a0 con 0x7f6c4412c900 2024-09-21T01:44:57.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.903+0000 7f6c49125640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f6c44133310 con 0x7f6c44128d40 2024-09-21T01:44:57.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.903+0000 7f6c42d9d640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6c44128d40 0x7f6c441c1930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:57.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.903+0000 7f6c4259c640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6c441296e0 0x7f6c441c1e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:57.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.903+0000 7f6c42d9d640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6c44128d40 0x7f6c441c1930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:55272/0 (socket says 172.21.15.105:55272) 2024-09-21T01:44:57.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.903+0000 7f6c42d9d640 1 -- 172.21.15.105:0/1638008050 learned_addr learned my addr 172.21.15.105:0/1638008050 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:57.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.903+0000 7f6c4359e640 1 --2- 172.21.15.105:0/1638008050 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6c4412c900 0x7f6c441c23b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:57.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.903+0000 7f6c42d9d640 1 -- 172.21.15.105:0/1638008050 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6c4412c900 msgr2=0x7f6c441c23b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:57.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.903+0000 7f6c42d9d640 1 --2- 172.21.15.105:0/1638008050 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6c4412c900 0x7f6c441c23b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:57.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.903+0000 7f6c42d9d640 1 -- 172.21.15.105:0/1638008050 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6c441296e0 msgr2=0x7f6c441c1e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:57.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.903+0000 7f6c42d9d640 1 --2- 172.21.15.105:0/1638008050 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6c441296e0 0x7f6c441c1e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:57.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.903+0000 7f6c42d9d640 1 -- 172.21.15.105:0/1638008050 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6c34009490 con 0x7f6c44128d40 2024-09-21T01:44:57.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.903+0000 7f6c4259c640 1 --2- 172.21.15.105:0/1638008050 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6c441296e0 0x7f6c441c1e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:57.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.903+0000 7f6c4359e640 1 --2- 172.21.15.105:0/1638008050 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6c4412c900 0x7f6c441c23b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:57.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.904+0000 7f6c42d9d640 1 --2- 172.21.15.105:0/1638008050 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6c44128d40 0x7f6c441c1930 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f6c3400ea60 tx=0x7f6c3400ef20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:57.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.904+0000 7f6c2b7fe640 1 -- 172.21.15.105:0/1638008050 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c3400cde0 con 0x7f6c44128d40 2024-09-21T01:44:57.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.904+0000 7f6c2b7fe640 1 -- 172.21.15.105:0/1638008050 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f6c34004020 con 0x7f6c44128d40 2024-09-21T01:44:57.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.904+0000 7f6c49125640 1 -- 172.21.15.105:0/1638008050 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6c38009490 con 0x7f6c44128d40 2024-09-21T01:44:57.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.904+0000 7f6c2b7fe640 1 -- 172.21.15.105:0/1638008050 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f6c34004410 con 0x7f6c44128d40 2024-09-21T01:44:57.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.904+0000 7f6c49125640 1 -- 172.21.15.105:0/1638008050 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6c441337b0 con 0x7f6c44128d40 2024-09-21T01:44:57.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.905+0000 7f6c2b7fe640 1 -- 172.21.15.105:0/1638008050 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f6c340035f0 con 0x7f6c44128d40 2024-09-21T01:44:57.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.905+0000 7f6c2b7fe640 1 -- 172.21.15.105:0/1638008050 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f6c34014070 con 0x7f6c44128d40 2024-09-21T01:44:57.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.905+0000 7f6c49125640 1 -- 172.21.15.105:0/1638008050 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6c441291a0 con 0x7f6c44128d40 2024-09-21T01:44:57.909 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:57.909+0000 7f6c2b7fe640 1 -- 172.21.15.105:0/1638008050 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f6c3403afa0 con 0x7f6c44128d40 2024-09-21T01:44:58.072 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.072+0000 7f6c49125640 1 -- 172.21.15.105:0/1638008050 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "mgr dump", "format": "json"} v 0) v1 -- 0x7f6c4406aeb0 con 0x7f6c44128d40 2024-09-21T01:44:58.074 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.074+0000 7f6c2b7fe640 1 -- 172.21.15.105:0/1638008050 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json"}]=0 v5) v1 ==== 74+0+107694 (secure 0 0 0) 0x7f6c3403e5c0 con 0x7f6c44128d40 2024-09-21T01:44:58.076 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:58.077 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.076+0000 7f6c49125640 1 -- 172.21.15.105:0/1638008050 >> v1:172.21.15.105:6816/3933313069 conn(0x7f6c30047560 legacy=0x7f6c30049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.077 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.076+0000 7f6c49125640 1 -- 172.21.15.105:0/1638008050 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6c44128d40 msgr2=0x7f6c441c1930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.077 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.076+0000 7f6c49125640 1 --2- 172.21.15.105:0/1638008050 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6c44128d40 0x7f6c441c1930 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f6c3400ea60 tx=0x7f6c3400ef20 comp rx=0 tx=0).stop 2024-09-21T01:44:58.077 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.076+0000 7f6c49125640 1 -- 172.21.15.105:0/1638008050 shutdown_connections 2024-09-21T01:44:58.077 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.076+0000 7f6c49125640 1 --2- 172.21.15.105:0/1638008050 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6c4412c900 0x7f6c441c23b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.077 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.076+0000 7f6c49125640 1 --2- 172.21.15.105:0/1638008050 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6c441296e0 0x7f6c441c1e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.077 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.076+0000 7f6c49125640 1 --2- 172.21.15.105:0/1638008050 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6c44128d40 0x7f6c441c1930 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.078 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.076+0000 7f6c49125640 1 -- 172.21.15.105:0/1638008050 >> 172.21.15.105:0/1638008050 conn(0x7f6c4409b7c0 msgr2=0x7f6c44128240 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:58.078 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.076+0000 7f6c49125640 1 -- 172.21.15.105:0/1638008050 shutdown_connections 2024-09-21T01:44:58.078 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.076+0000 7f6c49125640 1 -- 172.21.15.105:0/1638008050 wait complete. 2024-09-21T01:44:58.087 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":5,"flags":0,"active_gid":4112,"active_name":"y","active_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6816","nonce":3933313069}]},"active_addr":"172.21.15.105:6816/3933313069","active_change":"2024-09-21T01:44:40.342853+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":4106,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.105:0","nonce":3837276043}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.105:0","nonce":3561680891}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.105:0","nonce":2570087590}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.105:0","nonce":2810478014}]}]} 2024-09-21T01:44:58.089 INFO:tasks.ceph.ceph_manager.ceph:mgr available! 2024-09-21T01:44:58.089 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up 2024-09-21T01:44:58.089 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T01:44:58.187 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.186+0000 7fca156c9640 1 -- 172.21.15.105:0/2078994025 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fca10094060 msgr2=0x7fca1012d5c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.187 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.186+0000 7fca156c9640 1 --2- 172.21.15.105:0/2078994025 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fca10094060 0x7fca1012d5c0 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7fca04005fd0 tx=0x7fca0402a4b0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.187+0000 7fca156c9640 1 -- 172.21.15.105:0/2078994025 shutdown_connections 2024-09-21T01:44:58.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.187+0000 7fca156c9640 1 --2- 172.21.15.105:0/2078994025 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fca10094060 0x7fca1012d5c0 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.187+0000 7fca156c9640 1 --2- 172.21.15.105:0/2078994025 >> v2:172.21.15.105:3300/0 conn(0x7fca10093740 0x7fca10093b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.187+0000 7fca156c9640 1 -- 172.21.15.105:0/2078994025 >> 172.21.15.105:0/2078994025 conn(0x7fca10098760 msgr2=0x7fca10098b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:58.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.187+0000 7fca156c9640 1 -- 172.21.15.105:0/2078994025 shutdown_connections 2024-09-21T01:44:58.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.187+0000 7fca156c9640 1 -- 172.21.15.105:0/2078994025 wait complete. 2024-09-21T01:44:58.190 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.188+0000 7fca156c9640 1 Processor -- start 2024-09-21T01:44:58.190 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.188+0000 7fca156c9640 1 -- start start 2024-09-21T01:44:58.190 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.188+0000 7fca156c9640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fca10093740 0x7fca101c2090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.190 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.188+0000 7fca156c9640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fca10094060 0x7fca101c25d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.190 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.188+0000 7fca156c9640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fca1012db00 0x7fca101c2b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.191 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.188+0000 7fca156c9640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fca10070cb0 con 0x7fca1012db00 2024-09-21T01:44:58.191 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.188+0000 7fca156c9640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fca10090800 con 0x7fca10094060 2024-09-21T01:44:58.191 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.188+0000 7fca156c9640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fca10093570 con 0x7fca10093740 2024-09-21T01:44:58.191 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.188+0000 7fca0effd640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fca10093740 0x7fca101c2090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.191 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.188+0000 7fca0effd640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fca10093740 0x7fca101c2090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:55290/0 (socket says 172.21.15.105:55290) 2024-09-21T01:44:58.191 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.188+0000 7fca0e7fc640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fca10094060 0x7fca101c25d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.191 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.188+0000 7fca0effd640 1 -- 172.21.15.105:0/3844028850 learned_addr learned my addr 172.21.15.105:0/3844028850 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:58.191 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.188+0000 7fca0f7fe640 1 --2- 172.21.15.105:0/3844028850 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fca1012db00 0x7fca101c2b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.191 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.188+0000 7fca0f7fe640 1 -- 172.21.15.105:0/3844028850 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fca10093740 msgr2=0x7fca101c2090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.191 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.188+0000 7fca0f7fe640 1 --2- 172.21.15.105:0/3844028850 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fca10093740 0x7fca101c2090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.191 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.188+0000 7fca0f7fe640 1 -- 172.21.15.105:0/3844028850 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fca10094060 msgr2=0x7fca101c25d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.191 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.188+0000 7fca0f7fe640 1 --2- 172.21.15.105:0/3844028850 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fca10094060 0x7fca101c25d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.191 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.188+0000 7fca0f7fe640 1 -- 172.21.15.105:0/3844028850 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fca00009840 con 0x7fca1012db00 2024-09-21T01:44:58.192 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.188+0000 7fca0effd640 1 --2- 172.21.15.105:0/3844028850 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fca10093740 0x7fca101c2090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:58.192 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.189+0000 7fca0f7fe640 1 --2- 172.21.15.105:0/3844028850 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fca1012db00 0x7fca101c2b10 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7fca0800c310 tx=0x7fca0800c7d0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:58.192 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.189+0000 7fc9f77fe640 1 -- 172.21.15.105:0/3844028850 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fca080052d0 con 0x7fca1012db00 2024-09-21T01:44:58.192 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.189+0000 7fc9f77fe640 1 -- 172.21.15.105:0/3844028850 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fca080035f0 con 0x7fca1012db00 2024-09-21T01:44:58.192 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.189+0000 7fca156c9640 1 -- 172.21.15.105:0/3844028850 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fca04009560 con 0x7fca1012db00 2024-09-21T01:44:58.192 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.189+0000 7fc9f77fe640 1 -- 172.21.15.105:0/3844028850 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fca08003a00 con 0x7fca1012db00 2024-09-21T01:44:58.192 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.189+0000 7fca156c9640 1 -- 172.21.15.105:0/3844028850 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fca10092f10 con 0x7fca1012db00 2024-09-21T01:44:58.192 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.190+0000 7fca156c9640 1 -- 172.21.15.105:0/3844028850 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fca10093b20 con 0x7fca1012db00 2024-09-21T01:44:58.192 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.190+0000 7fc9f77fe640 1 -- 172.21.15.105:0/3844028850 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7fca08003b60 con 0x7fca1012db00 2024-09-21T01:44:58.192 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.190+0000 7fc9f77fe640 1 -- 172.21.15.105:0/3844028850 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7fca0805be90 con 0x7fca1012db00 2024-09-21T01:44:58.194 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.194+0000 7fc9f77fe640 1 -- 172.21.15.105:0/3844028850 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7fca08039be0 con 0x7fca1012db00 2024-09-21T01:44:58.332 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.331+0000 7fca156c9640 1 -- 172.21.15.105:0/3844028850 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fca1006ffd0 con 0x7fca1012db00 2024-09-21T01:44:58.332 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.332+0000 7fc9f77fe640 1 -- 172.21.15.105:0/3844028850 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v17) v1 ==== 74+0+8831 (secure 0 0 0) 0x7fca08017020 con 0x7fca1012db00 2024-09-21T01:44:58.332 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:58.332 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":17,"fsid":"355399d3-6490-4ad0-ac52-d793648ff276","created":"2024-09-21T01:44:37.282024+0000","modified":"2024-09-21T01:44:57.527109+0000","last_up_change":"2024-09-21T01:44:48.482537+0000","last_in_change":"2024-09-21T01:44:39.666350+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T01:44:50.380956+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T01:44:54.210449+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"03218bdb-ec48-44c7-bbf5-31cb53a3cea7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6800","nonce":4053719786}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6801","nonce":4053719786}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6803","nonce":4053719786}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6802","nonce":4053719786}]},"public_addr":"172.21.15.105:6800/4053719786","cluster_addr":"172.21.15.105:6801/4053719786","heartbeat_back_addr":"172.21.15.105:6803/4053719786","heartbeat_front_addr":"172.21.15.105:6802/4053719786","state":["exists","up"]},{"osd":1,"uuid":"5528ff44-922a-4e44-bb69-95fd9544d8dd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6812","nonce":1313742696}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6813","nonce":1313742696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6815","nonce":1313742696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6814","nonce":1313742696}]},"public_addr":"172.21.15.105:6812/1313742696","cluster_addr":"172.21.15.105:6813/1313742696","heartbeat_back_addr":"172.21.15.105:6815/1313742696","heartbeat_front_addr":"172.21.15.105:6814/1313742696","state":["exists","up"]},{"osd":2,"uuid":"df56ff35-71e9-4d46-8abb-84265eb6e1ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6804","nonce":2159605016}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6805","nonce":2159605016}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6807","nonce":2159605016}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6806","nonce":2159605016}]},"public_addr":"172.21.15.105:6804/2159605016","cluster_addr":"172.21.15.105:6805/2159605016","heartbeat_back_addr":"172.21.15.105:6807/2159605016","heartbeat_front_addr":"172.21.15.105:6806/2159605016","state":["exists","up"]},{"osd":3,"uuid":"012d9668-f1d3-4268-b0eb-15fa845ef17c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6808","nonce":2514115267}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6809","nonce":2514115267}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6811","nonce":2514115267}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6810","nonce":2514115267}]},"public_addr":"172.21.15.105:6808/2514115267","cluster_addr":"172.21.15.105:6809/2514115267","heartbeat_back_addr":"172.21.15.105:6811/2514115267","heartbeat_front_addr":"172.21.15.105:6810/2514115267","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.554795+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.869400+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.601687+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.822680+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T01:44:58.334 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.333+0000 7fca156c9640 1 -- 172.21.15.105:0/3844028850 >> v1:172.21.15.105:6816/3933313069 conn(0x7fc9fc047560 legacy=0x7fc9fc049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.334 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.334+0000 7fca156c9640 1 -- 172.21.15.105:0/3844028850 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fca1012db00 msgr2=0x7fca101c2b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.334 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.334+0000 7fca156c9640 1 --2- 172.21.15.105:0/3844028850 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fca1012db00 0x7fca101c2b10 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7fca0800c310 tx=0x7fca0800c7d0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.334 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.334+0000 7fca156c9640 1 -- 172.21.15.105:0/3844028850 shutdown_connections 2024-09-21T01:44:58.334 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.334+0000 7fca156c9640 1 --2- 172.21.15.105:0/3844028850 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fca1012db00 0x7fca101c2b10 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.334 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.334+0000 7fca156c9640 1 --2- 172.21.15.105:0/3844028850 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fca10094060 0x7fca101c25d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.335 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.334+0000 7fca156c9640 1 --2- 172.21.15.105:0/3844028850 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fca10093740 0x7fca101c2090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.335 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.334+0000 7fca156c9640 1 -- 172.21.15.105:0/3844028850 >> 172.21.15.105:0/3844028850 conn(0x7fca10098760 msgr2=0x7fca10089000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:58.335 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.334+0000 7fca156c9640 1 -- 172.21.15.105:0/3844028850 shutdown_connections 2024-09-21T01:44:58.335 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.334+0000 7fca156c9640 1 -- 172.21.15.105:0/3844028850 wait complete. 2024-09-21T01:44:58.345 INFO:tasks.ceph.ceph_manager.ceph:all up! 2024-09-21T01:44:58.345 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T01:44:58.443 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.442+0000 7f856360d640 1 -- 172.21.15.105:0/3654790439 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f855c12f710 msgr2=0x7f855c131b00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.443 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.442+0000 7f856360d640 1 --2- 172.21.15.105:0/3654790439 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f855c12f710 0x7f855c131b00 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f8554009be0 tx=0x7f855402a7c0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.443 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.442+0000 7f856360d640 1 -- 172.21.15.105:0/3654790439 shutdown_connections 2024-09-21T01:44:58.443 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.442+0000 7f856360d640 1 --2- 172.21.15.105:0/3654790439 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f855c12f710 0x7f855c131b00 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.443 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.442+0000 7f856360d640 1 --2- 172.21.15.105:0/3654790439 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f855c06f7f0 0x7f855c12f1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.443 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.442+0000 7f856360d640 1 -- 172.21.15.105:0/3654790439 >> 172.21.15.105:0/3654790439 conn(0x7f855c09cb90 msgr2=0x7f855c09cfa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:58.444 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.443+0000 7f856360d640 1 -- 172.21.15.105:0/3654790439 shutdown_connections 2024-09-21T01:44:58.444 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.443+0000 7f856360d640 1 -- 172.21.15.105:0/3654790439 wait complete. 2024-09-21T01:44:58.446 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.443+0000 7f856360d640 1 Processor -- start 2024-09-21T01:44:58.446 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.443+0000 7f856360d640 1 -- start start 2024-09-21T01:44:58.446 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f856360d640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f855c06ed00 0x7f855c0962b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.446 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f856360d640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f855c06f7f0 0x7f855c0967f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.446 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f856360d640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f855c12f710 0x7f855c12afa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.447 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f856360d640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f855c12d920 con 0x7f855c06f7f0 2024-09-21T01:44:58.447 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f856360d640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f855c12e100 con 0x7f855c12f710 2024-09-21T01:44:58.447 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f856360d640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f855c12e4d0 con 0x7f855c06ed00 2024-09-21T01:44:58.447 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f8560ba8640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f855c06f7f0 0x7f855c0967f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.447 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f85613a9640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f855c06ed00 0x7f855c0962b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.447 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f8561baa640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f855c12f710 0x7f855c12afa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.447 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f8560ba8640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f855c06f7f0 0x7f855c0967f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:41992/0 (socket says 172.21.15.105:41992) 2024-09-21T01:44:58.447 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f8560ba8640 1 -- 172.21.15.105:0/367390118 learned_addr learned my addr 172.21.15.105:0/367390118 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:58.447 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f8560ba8640 1 -- 172.21.15.105:0/367390118 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f855c06ed00 msgr2=0x7f855c0962b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.447 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f8560ba8640 1 --2- 172.21.15.105:0/367390118 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f855c06ed00 0x7f855c0962b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.447 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f8560ba8640 1 -- 172.21.15.105:0/367390118 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f855c12f710 msgr2=0x7f855c12afa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.447 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f8560ba8640 1 --2- 172.21.15.105:0/367390118 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f855c12f710 0x7f855c12afa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.447 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f8560ba8640 1 -- 172.21.15.105:0/367390118 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f854c009560 con 0x7f855c06f7f0 2024-09-21T01:44:58.448 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f8561baa640 1 --2- 172.21.15.105:0/367390118 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f855c12f710 0x7f855c12afa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:58.448 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f85613a9640 1 --2- 172.21.15.105:0/367390118 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f855c06ed00 0x7f855c0962b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:58.448 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f8560ba8640 1 --2- 172.21.15.105:0/367390118 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f855c06f7f0 0x7f855c0967f0 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f854c00e9c0 tx=0x7f854c00ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:58.448 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f8551ffb640 1 -- 172.21.15.105:0/367390118 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f854c00cde0 con 0x7f855c06f7f0 2024-09-21T01:44:58.448 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f8551ffb640 1 -- 172.21.15.105:0/367390118 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f854c004020 con 0x7f855c06f7f0 2024-09-21T01:44:58.448 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f856360d640 1 -- 172.21.15.105:0/367390118 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8554009840 con 0x7f855c06f7f0 2024-09-21T01:44:58.448 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.444+0000 7f8551ffb640 1 -- 172.21.15.105:0/367390118 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f854c004410 con 0x7f855c06f7f0 2024-09-21T01:44:58.448 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.445+0000 7f856360d640 1 -- 172.21.15.105:0/367390118 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f855c096d30 con 0x7f855c06f7f0 2024-09-21T01:44:58.448 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.445+0000 7f8551ffb640 1 -- 172.21.15.105:0/367390118 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f854c0035f0 con 0x7f855c06f7f0 2024-09-21T01:44:58.448 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.445+0000 7f856360d640 1 -- 172.21.15.105:0/367390118 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f855c123e20 con 0x7f855c06f7f0 2024-09-21T01:44:58.448 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.446+0000 7f8551ffb640 1 -- 172.21.15.105:0/367390118 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f854c014070 con 0x7f855c06f7f0 2024-09-21T01:44:58.449 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.449+0000 7f8551ffb640 1 -- 172.21.15.105:0/367390118 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f854c03af70 con 0x7f855c06f7f0 2024-09-21T01:44:58.586 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.586+0000 7f856360d640 1 -- 172.21.15.105:0/367390118 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f855c123480 con 0x7f855c06f7f0 2024-09-21T01:44:58.586 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.586+0000 7f8551ffb640 1 -- 172.21.15.105:0/367390118 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v17) v1 ==== 74+0+8831 (secure 0 0 0) 0x7f854c03e590 con 0x7f855c06f7f0 2024-09-21T01:44:58.587 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:44:58.587 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":17,"fsid":"355399d3-6490-4ad0-ac52-d793648ff276","created":"2024-09-21T01:44:37.282024+0000","modified":"2024-09-21T01:44:57.527109+0000","last_up_change":"2024-09-21T01:44:48.482537+0000","last_in_change":"2024-09-21T01:44:39.666350+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T01:44:50.380956+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T01:44:54.210449+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"03218bdb-ec48-44c7-bbf5-31cb53a3cea7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6800","nonce":4053719786}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6801","nonce":4053719786}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6803","nonce":4053719786}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6802","nonce":4053719786}]},"public_addr":"172.21.15.105:6800/4053719786","cluster_addr":"172.21.15.105:6801/4053719786","heartbeat_back_addr":"172.21.15.105:6803/4053719786","heartbeat_front_addr":"172.21.15.105:6802/4053719786","state":["exists","up"]},{"osd":1,"uuid":"5528ff44-922a-4e44-bb69-95fd9544d8dd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6812","nonce":1313742696}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6813","nonce":1313742696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6815","nonce":1313742696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6814","nonce":1313742696}]},"public_addr":"172.21.15.105:6812/1313742696","cluster_addr":"172.21.15.105:6813/1313742696","heartbeat_back_addr":"172.21.15.105:6815/1313742696","heartbeat_front_addr":"172.21.15.105:6814/1313742696","state":["exists","up"]},{"osd":2,"uuid":"df56ff35-71e9-4d46-8abb-84265eb6e1ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6804","nonce":2159605016}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6805","nonce":2159605016}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6807","nonce":2159605016}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6806","nonce":2159605016}]},"public_addr":"172.21.15.105:6804/2159605016","cluster_addr":"172.21.15.105:6805/2159605016","heartbeat_back_addr":"172.21.15.105:6807/2159605016","heartbeat_front_addr":"172.21.15.105:6806/2159605016","state":["exists","up"]},{"osd":3,"uuid":"012d9668-f1d3-4268-b0eb-15fa845ef17c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6808","nonce":2514115267}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6809","nonce":2514115267}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6811","nonce":2514115267}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6810","nonce":2514115267}]},"public_addr":"172.21.15.105:6808/2514115267","cluster_addr":"172.21.15.105:6809/2514115267","heartbeat_back_addr":"172.21.15.105:6811/2514115267","heartbeat_front_addr":"172.21.15.105:6810/2514115267","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.554795+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.869400+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.601687+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.822680+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T01:44:58.588 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.588+0000 7f856360d640 1 -- 172.21.15.105:0/367390118 >> v1:172.21.15.105:6816/3933313069 conn(0x7f8540047560 legacy=0x7f8540049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.588 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.588+0000 7f856360d640 1 -- 172.21.15.105:0/367390118 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f855c06f7f0 msgr2=0x7f855c0967f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.589 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.588+0000 7f856360d640 1 --2- 172.21.15.105:0/367390118 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f855c06f7f0 0x7f855c0967f0 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f854c00e9c0 tx=0x7f854c00ee80 comp rx=0 tx=0).stop 2024-09-21T01:44:58.589 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.588+0000 7f856360d640 1 -- 172.21.15.105:0/367390118 shutdown_connections 2024-09-21T01:44:58.589 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.588+0000 7f856360d640 1 --2- 172.21.15.105:0/367390118 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f855c12f710 0x7f855c12afa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.589 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.588+0000 7f856360d640 1 --2- 172.21.15.105:0/367390118 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f855c06f7f0 0x7f855c0967f0 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.589 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.588+0000 7f856360d640 1 --2- 172.21.15.105:0/367390118 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f855c06ed00 0x7f855c0962b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.589 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.588+0000 7f856360d640 1 -- 172.21.15.105:0/367390118 >> 172.21.15.105:0/367390118 conn(0x7f855c09cb90 msgr2=0x7f855c122190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:58.589 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.588+0000 7f856360d640 1 -- 172.21.15.105:0/367390118 shutdown_connections 2024-09-21T01:44:58.589 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.588+0000 7f856360d640 1 -- 172.21.15.105:0/367390118 wait complete. 2024-09-21T01:44:58.599 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 flush_pg_stats 2024-09-21T01:44:58.599 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 flush_pg_stats 2024-09-21T01:44:58.600 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 flush_pg_stats 2024-09-21T01:44:58.600 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 flush_pg_stats 2024-09-21T01:44:58.704 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.704+0000 7fd5b2ffd640 1 -- 172.21.15.105:0/3536564375 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd5a402ac90 con 0x7fd5b4097410 2024-09-21T01:44:58.705 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.704+0000 7ff249f65640 1 -- 172.21.15.105:0/132491337 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff24412c430 msgr2=0x7ff24412c810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.705 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.704+0000 7ff249f65640 1 --2- 172.21.15.105:0/132491337 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff24412c430 0x7ff24412c810 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7ff234009830 tx=0x7ff23402a3f0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.705 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.704+0000 7ff249f65640 1 -- 172.21.15.105:0/132491337 shutdown_connections 2024-09-21T01:44:58.705 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.704+0000 7ff249f65640 1 --2- 172.21.15.105:0/132491337 >> v2:172.21.15.105:3300/0 conn(0x7ff24409daa0 0x7ff2441288b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.705 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.704+0000 7ff249f65640 1 --2- 172.21.15.105:0/132491337 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff24412c430 0x7ff24412c810 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.705 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.704+0000 7ff249f65640 1 -- 172.21.15.105:0/132491337 >> 172.21.15.105:0/132491337 conn(0x7ff24409b7c0 msgr2=0x7ff24409bbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:58.705 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.704+0000 7fd5ba308640 1 -- 172.21.15.105:0/3536564375 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd5b4097410 msgr2=0x7fd5b40977f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.706 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.704+0000 7fd5ba308640 1 --2- 172.21.15.105:0/3536564375 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd5b4097410 0x7fd5b40977f0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7fd5a40071c0 tx=0x7fd5a4029fa0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.706 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.704+0000 7ff249f65640 1 -- 172.21.15.105:0/132491337 shutdown_connections 2024-09-21T01:44:58.706 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.704+0000 7ff249f65640 1 -- 172.21.15.105:0/132491337 wait complete. 2024-09-21T01:44:58.707 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.704+0000 7fd5ba308640 1 -- 172.21.15.105:0/3536564375 shutdown_connections 2024-09-21T01:44:58.707 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.704+0000 7fd5ba308640 1 --2- 172.21.15.105:0/3536564375 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd5b408c200 0x7fd5b406c460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.707 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.704+0000 7fd5ba308640 1 --2- 172.21.15.105:0/3536564375 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd5b4097410 0x7fd5b40977f0 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.707 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.705+0000 7fd5ba308640 1 -- 172.21.15.105:0/3536564375 >> 172.21.15.105:0/3536564375 conn(0x7fd5b40994b0 msgr2=0x7fd5b40998c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:58.707 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.705+0000 7fd5ba308640 1 -- 172.21.15.105:0/3536564375 shutdown_connections 2024-09-21T01:44:58.707 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.705+0000 7fd5ba308640 1 -- 172.21.15.105:0/3536564375 wait complete. 2024-09-21T01:44:58.707 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.705+0000 7f1d13ef7640 1 -- 172.21.15.105:0/3497032974 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1d0c125bf0 msgr2=0x7f1d0c125fd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.708 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.705+0000 7f1d13ef7640 1 --2- 172.21.15.105:0/3497032974 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1d0c125bf0 0x7f1d0c125fd0 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f1d08005fd0 tx=0x7f1d0802a4b0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.708 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.705+0000 7f62a9518640 1 -- 172.21.15.105:0/834511107 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f62a4092510 msgr2=0x7f62a40928f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.708 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.705+0000 7f1d13ef7640 1 -- 172.21.15.105:0/3497032974 shutdown_connections 2024-09-21T01:44:58.708 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.705+0000 7f62a9518640 1 --2- 172.21.15.105:0/834511107 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f62a4092510 0x7f62a40928f0 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f6294005fd0 tx=0x7f629402a4b0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.709 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.705+0000 7f62a9518640 1 -- 172.21.15.105:0/834511107 shutdown_connections 2024-09-21T01:44:58.709 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.705+0000 7f62a9518640 1 --2- 172.21.15.105:0/834511107 >> v2:172.21.15.105:3300/0 conn(0x7f62a4092e30 0x7f62a41283b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.709 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.705+0000 7f62a9518640 1 --2- 172.21.15.105:0/834511107 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f62a4092510 0x7f62a40928f0 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.709 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.705+0000 7f62a9518640 1 -- 172.21.15.105:0/834511107 >> 172.21.15.105:0/834511107 conn(0x7f62a409ada0 msgr2=0x7f62a409b1b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:58.709 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.705+0000 7f1d13ef7640 1 --2- 172.21.15.105:0/3497032974 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1d0c128ca0 0x7f1d0c1290c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.709 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.705+0000 7f1d13ef7640 1 --2- 172.21.15.105:0/3497032974 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1d0c125bf0 0x7f1d0c125fd0 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.709 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.705+0000 7f1d13ef7640 1 -- 172.21.15.105:0/3497032974 >> 172.21.15.105:0/3497032974 conn(0x7f1d0c0994b0 msgr2=0x7f1d0c0998c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:58.710 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.705+0000 7f62a9518640 1 -- 172.21.15.105:0/834511107 shutdown_connections 2024-09-21T01:44:58.710 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.705+0000 7f1d13ef7640 1 -- 172.21.15.105:0/3497032974 shutdown_connections 2024-09-21T01:44:58.710 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.705+0000 7f62a9518640 1 -- 172.21.15.105:0/834511107 wait complete. 2024-09-21T01:44:58.710 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.705+0000 7f1d13ef7640 1 -- 172.21.15.105:0/3497032974 wait complete. 2024-09-21T01:44:58.711 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.705+0000 7ff249f65640 1 Processor -- start 2024-09-21T01:44:58.711 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7fd5ba308640 1 Processor -- start 2024-09-21T01:44:58.711 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7ff249f65640 1 -- start start 2024-09-21T01:44:58.711 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7fd5ba308640 1 -- start start 2024-09-21T01:44:58.712 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7ff249f65640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff24409daa0 0x7ff2441c2020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.712 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7ff249f65640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff244128df0 0x7ff2441c2560 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.712 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7ff249f65640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff24412c430 0x7ff2441c2aa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.712 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7ff249f65640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff2441327b0 con 0x7ff244128df0 2024-09-21T01:44:58.712 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7ff249f65640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7ff244133620 con 0x7ff24409daa0 2024-09-21T01:44:58.712 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7ff249f65640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7ff244132f90 con 0x7ff24412c430 2024-09-21T01:44:58.712 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7f1d13ef7640 1 Processor -- start 2024-09-21T01:44:58.713 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7f62a9518640 1 Processor -- start 2024-09-21T01:44:58.713 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7f1d13ef7640 1 -- start start 2024-09-21T01:44:58.713 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7ff2437fe640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff24409daa0 0x7ff2441c2020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.714 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7fd5ba308640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd5b406c9a0 0x7fd5b4071130 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.714 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7fd5ba308640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd5b408c200 0x7fd5b4071670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.714 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7ff2437fe640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff24409daa0 0x7ff2441c2020 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:58750/0 (socket says 172.21.15.105:58750) 2024-09-21T01:44:58.714 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7ff2437fe640 1 -- 172.21.15.105:0/3396219262 learned_addr learned my addr 172.21.15.105:0/3396219262 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:58.714 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7ff243fff640 1 --2- 172.21.15.105:0/3396219262 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff24412c430 0x7ff2441c2aa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.714 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7ff242ffd640 1 --2- 172.21.15.105:0/3396219262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff244128df0 0x7ff2441c2560 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.715 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7fd5ba308640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd5b4097410 0x7fd5b4071bb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.715 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7fd5ba308640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd5b4129640 con 0x7fd5b406c9a0 2024-09-21T01:44:58.715 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7fd5ba308640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd5b4129e80 con 0x7fd5b408c200 2024-09-21T01:44:58.715 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7fd5ba308640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fd5b4128d80 con 0x7fd5b4097410 2024-09-21T01:44:58.716 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7fd5b37fe640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd5b408c200 0x7fd5b4071670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.716 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7fd5b3fff640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd5b406c9a0 0x7fd5b4071130 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.717 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7fd5b37fe640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd5b408c200 0x7fd5b4071670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:58762/0 (socket says 172.21.15.105:58762) 2024-09-21T01:44:58.717 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7fd5b3fff640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd5b406c9a0 0x7fd5b4071130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:42038/0 (socket says 172.21.15.105:42038) 2024-09-21T01:44:58.718 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7fd5b37fe640 1 -- 172.21.15.105:0/1505969046 learned_addr learned my addr 172.21.15.105:0/1505969046 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:58.718 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7fd5b88a5640 1 --2- 172.21.15.105:0/1505969046 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd5b4097410 0x7fd5b4071bb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.718 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7fd5b37fe640 1 -- 172.21.15.105:0/1505969046 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd5b4097410 msgr2=0x7fd5b4071bb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.718 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f1d13ef7640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1d0c125bf0 0x7f1d0c12fa90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.718 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f1d13ef7640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1d0c128ca0 0x7f1d0c12ffd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.718 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f1d13ef7640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f1d0c129600 0x7f1d0c130510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.718 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f1d13ef7640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d0c12f840 con 0x7f1d0c128ca0 2024-09-21T01:44:58.718 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f1d13ef7640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f1d0c134210 con 0x7f1d0c129600 2024-09-21T01:44:58.718 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f1d13ef7640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f1d0c125260 con 0x7f1d0c125bf0 2024-09-21T01:44:58.718 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f1d12494640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f1d0c129600 0x7f1d0c130510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.718 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f1d11c93640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1d0c125bf0 0x7f1d0c12fa90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.719 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f1d12494640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f1d0c129600 0x7f1d0c130510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:58774/0 (socket says 172.21.15.105:58774) 2024-09-21T01:44:58.719 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f1d11492640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1d0c128ca0 0x7f1d0c12ffd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.719 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f1d11c93640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1d0c125bf0 0x7f1d0c12fa90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:55354/0 (socket says 172.21.15.105:55354) 2024-09-21T01:44:58.719 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f1d12494640 1 -- 172.21.15.105:0/2359956718 learned_addr learned my addr 172.21.15.105:0/2359956718 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:58.719 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7fd5b37fe640 1 --2- 172.21.15.105:0/1505969046 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd5b4097410 0x7fd5b4071bb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.719 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7fd5b37fe640 1 -- 172.21.15.105:0/1505969046 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd5b406c9a0 msgr2=0x7fd5b4071130 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.719 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7fd5b37fe640 1 --2- 172.21.15.105:0/1505969046 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd5b406c9a0 0x7fd5b4071130 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.719 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7fd5b37fe640 1 -- 172.21.15.105:0/1505969046 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd5a400d040 con 0x7fd5b408c200 2024-09-21T01:44:58.719 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7fd5b88a5640 1 --2- 172.21.15.105:0/1505969046 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd5b4097410 0x7fd5b4071bb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T01:44:58.720 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.706+0000 7f62a9518640 1 -- start start 2024-09-21T01:44:58.720 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7fd5b3fff640 1 --2- 172.21.15.105:0/1505969046 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd5b406c9a0 0x7fd5b4071130 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello state changed while learned_addr, mark_down or replacing must be happened just now 2024-09-21T01:44:58.720 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7fd5b37fe640 1 --2- 172.21.15.105:0/1505969046 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd5b408c200 0x7fd5b4071670 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7fd5a8009960 tx=0x7fd5a8009e20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:58.720 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f62a9518640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f62a4092e30 0x7f62a4132810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.720 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f62a9518640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f62a41288f0 0x7f62a4132d50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.721 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f62a9518640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f62a4133290 0x7f62a41cc830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.721 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f62a9518640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f62a4122d90 con 0x7f62a41288f0 2024-09-21T01:44:58.721 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f62a9518640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f62a4071f70 con 0x7f62a4092e30 2024-09-21T01:44:58.721 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f62a9518640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f62a4091a80 con 0x7f62a4133290 2024-09-21T01:44:58.721 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f62a37fe640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f62a4133290 0x7f62a41cc830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.721 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f1d12494640 1 -- 172.21.15.105:0/2359956718 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1d0c125bf0 msgr2=0x7f1d0c12fa90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.721 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f1d12494640 1 --2- 172.21.15.105:0/2359956718 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1d0c125bf0 0x7f1d0c12fa90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.721 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f1d12494640 1 -- 172.21.15.105:0/2359956718 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1d0c128ca0 msgr2=0x7f1d0c12ffd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.721 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f1d12494640 1 --2- 172.21.15.105:0/2359956718 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1d0c128ca0 0x7f1d0c12ffd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.721 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f1d12494640 1 -- 172.21.15.105:0/2359956718 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1d08009560 con 0x7f1d0c129600 2024-09-21T01:44:58.722 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f62a37fe640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f62a4133290 0x7f62a41cc830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:55370/0 (socket says 172.21.15.105:55370) 2024-09-21T01:44:58.722 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7ff2437fe640 1 -- 172.21.15.105:0/3396219262 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff24412c430 msgr2=0x7ff2441c2aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.722 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7ff2437fe640 1 --2- 172.21.15.105:0/3396219262 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff24412c430 0x7ff2441c2aa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.722 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7ff2437fe640 1 -- 172.21.15.105:0/3396219262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff244128df0 msgr2=0x7ff2441c2560 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.722 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7ff2437fe640 1 --2- 172.21.15.105:0/3396219262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff244128df0 0x7ff2441c2560 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.723 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7ff2437fe640 1 -- 172.21.15.105:0/3396219262 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff234009490 con 0x7ff24409daa0 2024-09-21T01:44:58.723 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7ff2437fe640 1 --2- 172.21.15.105:0/3396219262 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff24409daa0 0x7ff2441c2020 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7ff234000c00 tx=0x7ff234003be0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:58.723 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7fd5b0ff9640 1 -- 172.21.15.105:0/1505969046 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd5a8012070 con 0x7fd5b408c200 2024-09-21T01:44:58.723 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7f1d12494640 1 --2- 172.21.15.105:0/2359956718 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f1d0c129600 0x7f1d0c130510 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f1d0400c7a0 tx=0x7f1d0400cc60 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:58.723 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.707+0000 7f62a37fe640 1 -- 172.21.15.105:0/792116964 learned_addr learned my addr 172.21.15.105:0/792116964 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:58.724 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7fd5b0ff9640 1 -- 172.21.15.105:0/1505969046 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fd5a800ca60 con 0x7fd5b408c200 2024-09-21T01:44:58.724 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7fd5b0ff9640 1 -- 172.21.15.105:0/1505969046 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd5a800d670 con 0x7fd5b408c200 2024-09-21T01:44:58.724 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7f62a37fe640 1 -- 172.21.15.105:0/792116964 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f62a4092e30 msgr2=0x7f62a4132810 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:44:58.724 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7ff23bfff640 1 -- 172.21.15.105:0/3396219262 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff23403c830 con 0x7ff24409daa0 2024-09-21T01:44:58.725 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7ff23bfff640 1 -- 172.21.15.105:0/3396219262 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7ff23403ca80 con 0x7ff24409daa0 2024-09-21T01:44:58.725 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7fd5ba308640 1 -- 172.21.15.105:0/1505969046 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd5b4128130 con 0x7fd5b408c200 2024-09-21T01:44:58.725 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7f62a37fe640 1 --2- 172.21.15.105:0/792116964 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f62a4092e30 0x7f62a4132810 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.725 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7fd5ba308640 1 -- 172.21.15.105:0/1505969046 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd5b41289d0 con 0x7fd5b408c200 2024-09-21T01:44:58.726 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7f62a37fe640 1 -- 172.21.15.105:0/792116964 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f62a41288f0 msgr2=0x7f62a4132d50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.726 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7f62a37fe640 1 --2- 172.21.15.105:0/792116964 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f62a41288f0 0x7f62a4132d50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.726 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7f62a37fe640 1 -- 172.21.15.105:0/792116964 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6294009560 con 0x7f62a4133290 2024-09-21T01:44:58.726 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7f62a37fe640 1 --2- 172.21.15.105:0/792116964 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f62a4133290 0x7f62a41cc830 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f629c00c7a0 tx=0x7f629c00cc60 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:58.727 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7f629b7fe640 1 -- 172.21.15.105:0/792116964 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f629c003b70 con 0x7f62a4133290 2024-09-21T01:44:58.727 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7f629b7fe640 1 -- 172.21.15.105:0/792116964 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f629c003cf0 con 0x7f62a4133290 2024-09-21T01:44:58.727 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7f62a9518640 1 -- 172.21.15.105:0/792116964 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f62a41230a0 con 0x7f62a4133290 2024-09-21T01:44:58.727 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7ff249f65640 1 -- 172.21.15.105:0/3396219262 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff244133360 con 0x7ff24409daa0 2024-09-21T01:44:58.727 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7ff249f65640 1 -- 172.21.15.105:0/3396219262 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff24412fdf0 con 0x7ff24409daa0 2024-09-21T01:44:58.727 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7ff23bfff640 1 -- 172.21.15.105:0/3396219262 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7ff23403ce70 con 0x7ff24409daa0 2024-09-21T01:44:58.728 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7f62a9518640 1 -- 172.21.15.105:0/792116964 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f62a41226f0 con 0x7f62a4133290 2024-09-21T01:44:58.728 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.709+0000 7f629b7fe640 1 -- 172.21.15.105:0/792116964 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f629c00aab0 con 0x7f62a4133290 2024-09-21T01:44:58.728 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7f1d027fc640 1 -- 172.21.15.105:0/2359956718 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d04003b70 con 0x7f1d0c129600 2024-09-21T01:44:58.728 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7f1d13ef7640 1 -- 172.21.15.105:0/2359956718 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1d0c072480 con 0x7f1d0c129600 2024-09-21T01:44:58.728 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7f1d027fc640 1 -- 172.21.15.105:0/2359956718 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f1d04003cf0 con 0x7f1d0c129600 2024-09-21T01:44:58.728 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.708+0000 7f1d027fc640 1 -- 172.21.15.105:0/2359956718 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f1d0400aab0 con 0x7f1d0c129600 2024-09-21T01:44:58.728 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.709+0000 7f1d13ef7640 1 -- 172.21.15.105:0/2359956718 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1d0c12edd0 con 0x7f1d0c129600 2024-09-21T01:44:58.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.709+0000 7fd5b0ff9640 1 -- 172.21.15.105:0/1505969046 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7fd5a8004020 con 0x7fd5b408c200 2024-09-21T01:44:58.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.709+0000 7f62a9518640 1 -- 172.21.15.105:0/792116964 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f6288000fc0 con 0x7f62a4133290 2024-09-21T01:44:58.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.710+0000 7fd5b0ff9640 1 -- 172.21.15.105:0/1505969046 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7fd5a805c4b0 con 0x7fd5b408c200 2024-09-21T01:44:58.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.710+0000 7fd5ba308640 1 -- 172.21.15.105:0/1505969046 --> v1:172.21.15.105:6808/2514115267 -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7fd5b4124930 con 0x7fd5b40608f0 2024-09-21T01:44:58.730 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.710+0000 7f1d027fc640 1 -- 172.21.15.105:0/2359956718 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f1d0400ac10 con 0x7f1d0c129600 2024-09-21T01:44:58.730 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.710+0000 7ff249f65640 1 -- 172.21.15.105:0/3396219262 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7ff24409bbf0 con 0x7ff24409daa0 2024-09-21T01:44:58.730 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.710+0000 7f629b7fe640 1 -- 172.21.15.105:0/792116964 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f629c00ac10 con 0x7f62a4133290 2024-09-21T01:44:58.730 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.710+0000 7f1d027fc640 1 -- 172.21.15.105:0/2359956718 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f1d0405c730 con 0x7f1d0c129600 2024-09-21T01:44:58.731 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.710+0000 7ff23bfff640 1 -- 172.21.15.105:0/3396219262 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7ff234044400 con 0x7ff24409daa0 2024-09-21T01:44:58.731 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.711+0000 7f1d13ef7640 1 -- 172.21.15.105:0/2359956718 --> v1:172.21.15.105:6812/1313742696 -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f1d0c12eb40 con 0x7f1d0c131980 2024-09-21T01:44:58.731 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.711+0000 7f629b7fe640 1 -- 172.21.15.105:0/792116964 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f629c05c4d0 con 0x7f62a4133290 2024-09-21T01:44:58.732 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.711+0000 7ff23bfff640 1 -- 172.21.15.105:0/3396219262 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7ff23407df10 con 0x7ff24409daa0 2024-09-21T01:44:58.732 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.711+0000 7ff23bfff640 1 -- 172.21.15.105:0/3396219262 --> v1:172.21.15.105:6800/4053719786 -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7ff2240519b0 con 0x7ff22404dcc0 2024-09-21T01:44:58.732 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.711+0000 7ff23bfff640 1 -- 172.21.15.105:0/3396219262 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_get_version_reply(handle=1 version=17) v2 ==== 24+0+0 (secure 0 0 0) 0x7ff234083500 con 0x7ff24409daa0 2024-09-21T01:44:58.732 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.711+0000 7fd5b0ff9640 1 -- 172.21.15.105:0/1505969046 <== osd.3 v1:172.21.15.105:6808/2514115267 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+26118 (unknown 0 0 1686565167) 0x7fd5b4124930 con 0x7fd5b40608f0 2024-09-21T01:44:58.732 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.711+0000 7f629b7fe640 1 -- 172.21.15.105:0/792116964 --> v1:172.21.15.105:6804/2159605016 -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f6284051960 con 0x7f628404dc70 2024-09-21T01:44:58.732 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.711+0000 7f629b7fe640 1 -- 172.21.15.105:0/792116964 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_get_version_reply(handle=1 version=17) v2 ==== 24+0+0 (secure 0 0 0) 0x7f629c05eab0 con 0x7f62a4133290 2024-09-21T01:44:58.733 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.712+0000 7ff23bfff640 1 -- 172.21.15.105:0/3396219262 <== osd.0 v1:172.21.15.105:6800/4053719786 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+26118 (unknown 0 0 1686565167) 0x7ff2240519b0 con 0x7ff22404dcc0 2024-09-21T01:44:58.733 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.712+0000 7f1d027fc640 1 -- 172.21.15.105:0/2359956718 <== osd.1 v1:172.21.15.105:6812/1313742696 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+26118 (unknown 0 0 1686565167) 0x7f1d0c12eb40 con 0x7f1d0c131980 2024-09-21T01:44:58.733 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.713+0000 7f629b7fe640 1 -- 172.21.15.105:0/792116964 <== osd.2 v1:172.21.15.105:6804/2159605016 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+26118 (unknown 0 0 1686565167) 0x7f6284051960 con 0x7f628404dc70 2024-09-21T01:44:58.734 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.725+0000 7fd5ba308640 1 -- 172.21.15.105:0/1505969046 --> v1:172.21.15.105:6808/2514115267 -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7fd5b406d2b0 con 0x7fd5b40608f0 2024-09-21T01:44:58.734 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.725+0000 7fd5b0ff9640 1 -- 172.21.15.105:0/1505969046 <== osd.3 v1:172.21.15.105:6808/2514115267 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (unknown 0 0 3659784045) 0x7fd5ac003cc0 con 0x7fd5b40608f0 2024-09-21T01:44:58.735 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.726+0000 7fd5ba308640 1 -- 172.21.15.105:0/1505969046 >> v1:172.21.15.105:6808/2514115267 conn(0x7fd5b40608f0 legacy=0x7fd5b4060cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.735 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.726+0000 7fd5ba308640 1 -- 172.21.15.105:0/1505969046 >> v1:172.21.15.105:6816/3933313069 conn(0x7fd5a0047630 legacy=0x7fd5a0049af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.735 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.726+0000 7fd5ba308640 1 -- 172.21.15.105:0/1505969046 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd5b408c200 msgr2=0x7fd5b4071670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.735 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.726+0000 7fd5ba308640 1 --2- 172.21.15.105:0/1505969046 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd5b408c200 0x7fd5b4071670 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7fd5a8009960 tx=0x7fd5a8009e20 comp rx=0 tx=0).stop 2024-09-21T01:44:58.735 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.726+0000 7fd5b88a5640 1 -- 172.21.15.105:0/1505969046 reap_dead start 2024-09-21T01:44:58.735 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.726+0000 7fd5ba308640 1 -- 172.21.15.105:0/1505969046 shutdown_connections 2024-09-21T01:44:58.735 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.726+0000 7fd5ba308640 1 -- 172.21.15.105:0/1505969046 >> 172.21.15.105:0/1505969046 conn(0x7fd5b40994b0 msgr2=0x7fd5b41216e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:58.735 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.726+0000 7fd5ba308640 1 -- 172.21.15.105:0/1505969046 shutdown_connections 2024-09-21T01:44:58.735 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.726+0000 7fd5ba308640 1 -- 172.21.15.105:0/1505969046 wait complete. 2024-09-21T01:44:58.736 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.726+0000 7f62a9518640 1 -- 172.21.15.105:0/792116964 --> v1:172.21.15.105:6804/2159605016 -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f6288002c80 con 0x7f628404dc70 2024-09-21T01:44:58.736 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.727+0000 7f629b7fe640 1 -- 172.21.15.105:0/792116964 <== osd.2 v1:172.21.15.105:6804/2159605016 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (unknown 0 0 3659784045) 0x7f6288002c80 con 0x7f628404dc70 2024-09-21T01:44:58.737 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.727+0000 7f62a9518640 1 -- 172.21.15.105:0/792116964 >> v1:172.21.15.105:6804/2159605016 conn(0x7f628404dc70 legacy=0x7f62840500f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.737 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.727+0000 7f62a9518640 1 -- 172.21.15.105:0/792116964 >> v1:172.21.15.105:6816/3933313069 conn(0x7f62840475e0 legacy=0x7f6284049aa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.737 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.727+0000 7f62a9518640 1 -- 172.21.15.105:0/792116964 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f62a4133290 msgr2=0x7f62a41cc830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.737 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.727+0000 7f62a9518640 1 --2- 172.21.15.105:0/792116964 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f62a4133290 0x7f62a41cc830 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f629c00c7a0 tx=0x7f629c00cc60 comp rx=0 tx=0).stop 2024-09-21T01:44:58.737 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.727+0000 7f62a37fe640 1 -- 172.21.15.105:0/792116964 reap_dead start 2024-09-21T01:44:58.737 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.727+0000 7f62a9518640 1 -- 172.21.15.105:0/792116964 shutdown_connections 2024-09-21T01:44:58.737 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.727+0000 7f62a9518640 1 -- 172.21.15.105:0/792116964 >> 172.21.15.105:0/792116964 conn(0x7f62a409ada0 msgr2=0x7f62a406ce90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:58.737 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.727+0000 7f62a9518640 1 -- 172.21.15.105:0/792116964 shutdown_connections 2024-09-21T01:44:58.738 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.727+0000 7f62a9518640 1 -- 172.21.15.105:0/792116964 wait complete. 2024-09-21T01:44:58.738 INFO:teuthology.orchestra.run.smithi105.stdout:38654705668 2024-09-21T01:44:58.739 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T01:44:58.739 INFO:teuthology.orchestra.run.smithi105.stdout:38654705668 2024-09-21T01:44:58.740 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T01:44:58.743 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.743+0000 7f1d13ef7640 1 -- 172.21.15.105:0/2359956718 --> v1:172.21.15.105:6812/1313742696 -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f1d0c1339c0 con 0x7f1d0c131980 2024-09-21T01:44:58.743 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.743+0000 7ff249f65640 1 -- 172.21.15.105:0/3396219262 --> v1:172.21.15.105:6800/4053719786 -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7ff244134d30 con 0x7ff22404dcc0 2024-09-21T01:44:58.745 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.743+0000 7ff23bfff640 1 -- 172.21.15.105:0/3396219262 <== osd.0 v1:172.21.15.105:6800/4053719786 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (unknown 0 0 675830894) 0x7ff244134d30 con 0x7ff22404dcc0 2024-09-21T01:44:58.745 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.743+0000 7ff249f65640 1 -- 172.21.15.105:0/3396219262 >> v1:172.21.15.105:6800/4053719786 conn(0x7ff22404dcc0 legacy=0x7ff224050140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.745 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.744+0000 7ff249f65640 1 -- 172.21.15.105:0/3396219262 >> v1:172.21.15.105:6816/3933313069 conn(0x7ff224047630 legacy=0x7ff224049af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.745 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.744+0000 7ff249f65640 1 -- 172.21.15.105:0/3396219262 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff24409daa0 msgr2=0x7ff2441c2020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.745 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.744+0000 7ff249f65640 1 --2- 172.21.15.105:0/3396219262 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff24409daa0 0x7ff2441c2020 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7ff234000c00 tx=0x7ff234003be0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.745 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.744+0000 7ff243fff640 1 -- 172.21.15.105:0/3396219262 reap_dead start 2024-09-21T01:44:58.745 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.744+0000 7ff249f65640 1 -- 172.21.15.105:0/3396219262 shutdown_connections 2024-09-21T01:44:58.745 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.744+0000 7ff249f65640 1 -- 172.21.15.105:0/3396219262 >> 172.21.15.105:0/3396219262 conn(0x7ff24409b7c0 msgr2=0x7ff244127f80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:58.746 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.743+0000 7f1d027fc640 1 -- 172.21.15.105:0/2359956718 <== osd.1 v1:172.21.15.105:6812/1313742696 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (unknown 0 0 3659784045) 0x7f1d0c1339c0 con 0x7f1d0c131980 2024-09-21T01:44:58.746 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.743+0000 7f1d13ef7640 1 -- 172.21.15.105:0/2359956718 >> v1:172.21.15.105:6812/1313742696 conn(0x7f1d0c131980 legacy=0x7f1d0c060e00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.746 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.744+0000 7f1d13ef7640 1 -- 172.21.15.105:0/2359956718 >> v1:172.21.15.105:6816/3933313069 conn(0x7f1cf0047560 legacy=0x7f1cf0049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.746 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.744+0000 7f1d13ef7640 1 -- 172.21.15.105:0/2359956718 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f1d0c129600 msgr2=0x7f1d0c130510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.746 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.744+0000 7f1d13ef7640 1 --2- 172.21.15.105:0/2359956718 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f1d0c129600 0x7f1d0c130510 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f1d0400c7a0 tx=0x7f1d0400cc60 comp rx=0 tx=0).stop 2024-09-21T01:44:58.746 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.744+0000 7f1d12494640 1 -- 172.21.15.105:0/2359956718 reap_dead start 2024-09-21T01:44:58.746 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.744+0000 7f1d13ef7640 1 -- 172.21.15.105:0/2359956718 shutdown_connections 2024-09-21T01:44:58.746 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.744+0000 7f1d13ef7640 1 -- 172.21.15.105:0/2359956718 >> 172.21.15.105:0/2359956718 conn(0x7f1d0c0994b0 msgr2=0x7f1d0c08b240 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:58.746 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.744+0000 7f1d13ef7640 1 -- 172.21.15.105:0/2359956718 shutdown_connections 2024-09-21T01:44:58.747 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.744+0000 7f1d13ef7640 1 -- 172.21.15.105:0/2359956718 wait complete. 2024-09-21T01:44:58.747 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.744+0000 7ff249f65640 1 -- 172.21.15.105:0/3396219262 shutdown_connections 2024-09-21T01:44:58.747 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.744+0000 7ff249f65640 1 -- 172.21.15.105:0/3396219262 wait complete. 2024-09-21T01:44:58.756 INFO:teuthology.orchestra.run.smithi105.stdout:38654705668 2024-09-21T01:44:58.757 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T01:44:58.757 INFO:teuthology.orchestra.run.smithi105.stdout:38654705669 2024-09-21T01:44:58.758 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T01:44:58.847 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.846+0000 7f44a119d640 1 -- 172.21.15.105:0/3641966275 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f449c128500 msgr2=0x7f449c12eea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.847 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.846+0000 7f44a119d640 1 --2- 172.21.15.105:0/3641966275 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f449c128500 0x7f449c12eea0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f4490009860 tx=0x7f449002a4b0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.847 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.846+0000 7f44a119d640 1 -- 172.21.15.105:0/3641966275 shutdown_connections 2024-09-21T01:44:58.847 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.846+0000 7f44a119d640 1 --2- 172.21.15.105:0/3641966275 >> v2:172.21.15.105:3300/0 conn(0x7f449c12f3e0 0x7f449c1317d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.847 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.846+0000 7f44a119d640 1 --2- 172.21.15.105:0/3641966275 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f449c128500 0x7f449c12eea0 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.847 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.846+0000 7f44a119d640 1 -- 172.21.15.105:0/3641966275 >> 172.21.15.105:0/3641966275 conn(0x7f449c09c710 msgr2=0x7f449c09cb20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:58.847 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.846+0000 7f44a119d640 1 -- 172.21.15.105:0/3641966275 shutdown_connections 2024-09-21T01:44:58.848 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.846+0000 7f44a119d640 1 -- 172.21.15.105:0/3641966275 wait complete. 2024-09-21T01:44:58.848 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.847+0000 7f217d44f640 1 -- 172.21.15.105:0/2332026544 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f217812e8c0 msgr2=0x7f2178130cb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.848 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.847+0000 7f217d44f640 1 --2- 172.21.15.105:0/2332026544 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f217812e8c0 0x7f2178130cb0 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f2170005fd0 tx=0x7f217002a780 comp rx=0 tx=0).stop 2024-09-21T01:44:58.849 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.847+0000 7f217d44f640 1 -- 172.21.15.105:0/2332026544 shutdown_connections 2024-09-21T01:44:58.849 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.847+0000 7f217d44f640 1 --2- 172.21.15.105:0/2332026544 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f217812e8c0 0x7f2178130cb0 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.849 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.847+0000 7f217d44f640 1 --2- 172.21.15.105:0/2332026544 >> v2:172.21.15.105:3300/0 conn(0x7f217812bf90 0x7f217812e380 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.849 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.847+0000 7f217d44f640 1 -- 172.21.15.105:0/2332026544 >> 172.21.15.105:0/2332026544 conn(0x7f2178099740 msgr2=0x7f2178099b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:58.849 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.847+0000 7f217d44f640 1 -- 172.21.15.105:0/2332026544 shutdown_connections 2024-09-21T01:44:58.849 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.847+0000 7f217d44f640 1 -- 172.21.15.105:0/2332026544 wait complete. 2024-09-21T01:44:58.849 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.847+0000 7f44a119d640 1 Processor -- start 2024-09-21T01:44:58.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.847+0000 7f217d44f640 1 Processor -- start 2024-09-21T01:44:58.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f217d44f640 1 -- start start 2024-09-21T01:44:58.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f44a119d640 1 -- start start 2024-09-21T01:44:58.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f217d44f640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2178128f80 0x7f21781200d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f217d44f640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f217812bf90 0x7f2178120610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f217d44f640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f217812e8c0 0x7f2178120b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f217d44f640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2178122b10 con 0x7f217812bf90 2024-09-21T01:44:58.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f217d44f640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f21780704c0 con 0x7f217812e8c0 2024-09-21T01:44:58.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f217d44f640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f2178121bf0 con 0x7f2178128f80 2024-09-21T01:44:58.852 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f44a119d640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f449c127a10 0x7f449c070690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.852 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f44a119d640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f449c128500 0x7f449c070bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.852 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f44a119d640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f449c12f3e0 0x7f449c071110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.852 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f44a119d640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f449c12ec90 con 0x7f449c128500 2024-09-21T01:44:58.852 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f44a119d640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f449c071f50 con 0x7f449c127a10 2024-09-21T01:44:58.852 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f44a119d640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f449c1243e0 con 0x7f449c12f3e0 2024-09-21T01:44:58.852 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f449b7fe640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f449c128500 0x7f449c070bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.852 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f449b7fe640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f449c128500 0x7f449c070bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:42098/0 (socket says 172.21.15.105:42098) 2024-09-21T01:44:58.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f21777fe640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f217812e8c0 0x7f2178120b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f2176ffd640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2178128f80 0x7f21781200d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f216ffff640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f217812bf90 0x7f2178120610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f21777fe640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f217812e8c0 0x7f2178120b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:58816/0 (socket says 172.21.15.105:58816) 2024-09-21T01:44:58.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f21777fe640 1 -- 172.21.15.105:0/477009955 learned_addr learned my addr 172.21.15.105:0/477009955 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:58.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f449b7fe640 1 -- 172.21.15.105:0/811757463 learned_addr learned my addr 172.21.15.105:0/811757463 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:58.854 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f449bfff640 1 --2- 172.21.15.105:0/811757463 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f449c127a10 0x7f449c070690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.854 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.848+0000 7f44a099c640 1 --2- 172.21.15.105:0/811757463 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f449c12f3e0 0x7f449c071110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.854 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.849+0000 7f449b7fe640 1 -- 172.21.15.105:0/811757463 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f449c12f3e0 msgr2=0x7f449c071110 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.854 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.849+0000 7f21777fe640 1 -- 172.21.15.105:0/477009955 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2178128f80 msgr2=0x7f21781200d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.854 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.849+0000 7f21777fe640 1 --2- 172.21.15.105:0/477009955 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2178128f80 0x7f21781200d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.854 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.849+0000 7f21777fe640 1 -- 172.21.15.105:0/477009955 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f217812bf90 msgr2=0x7f2178120610 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.854 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.849+0000 7f21777fe640 1 --2- 172.21.15.105:0/477009955 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f217812bf90 0x7f2178120610 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.854 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.849+0000 7f21777fe640 1 -- 172.21.15.105:0/477009955 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2170009840 con 0x7f217812e8c0 2024-09-21T01:44:58.855 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.849+0000 7f449b7fe640 1 --2- 172.21.15.105:0/811757463 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f449c12f3e0 0x7f449c071110 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.855 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.849+0000 7f449b7fe640 1 -- 172.21.15.105:0/811757463 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f449c127a10 msgr2=0x7f449c070690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.855 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.849+0000 7f449b7fe640 1 --2- 172.21.15.105:0/811757463 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f449c127a10 0x7f449c070690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.855 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.849+0000 7f21777fe640 1 --2- 172.21.15.105:0/477009955 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f217812e8c0 0x7f2178120b50 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f2170009b30 tx=0x7f2170003c10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:58.856 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.849+0000 7f449b7fe640 1 -- 172.21.15.105:0/811757463 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4490009490 con 0x7f449c128500 2024-09-21T01:44:58.856 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.849+0000 7f449bfff640 1 --2- 172.21.15.105:0/811757463 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f449c127a10 0x7f449c070690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T01:44:58.856 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.849+0000 7f44a099c640 1 --2- 172.21.15.105:0/811757463 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f449c12f3e0 0x7f449c071110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T01:44:58.856 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.849+0000 7f449b7fe640 1 --2- 172.21.15.105:0/811757463 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f449c128500 0x7f449c070bd0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f4490000c00 tx=0x7f4490003c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:58.856 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.849+0000 7f4498ff9640 1 -- 172.21.15.105:0/811757463 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4490005cb0 con 0x7f449c128500 2024-09-21T01:44:58.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.849+0000 7f216f7fe640 1 -- 172.21.15.105:0/477009955 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f217003f070 con 0x7f217812e8c0 2024-09-21T01:44:58.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.849+0000 7f216f7fe640 1 -- 172.21.15.105:0/477009955 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f2170003e40 con 0x7f217812e8c0 2024-09-21T01:44:58.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.849+0000 7f216f7fe640 1 -- 172.21.15.105:0/477009955 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f21700044b0 con 0x7f217812e8c0 2024-09-21T01:44:58.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.849+0000 7f217d44f640 1 -- 172.21.15.105:0/477009955 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f21781216e0 con 0x7f217812e8c0 2024-09-21T01:44:58.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.850+0000 7f217d44f640 1 -- 172.21.15.105:0/477009955 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f21781bfdb0 con 0x7f217812e8c0 2024-09-21T01:44:58.858 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.849+0000 7f4498ff9640 1 -- 172.21.15.105:0/811757463 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f449003c710 con 0x7f449c128500 2024-09-21T01:44:58.858 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.849+0000 7f4498ff9640 1 -- 172.21.15.105:0/811757463 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f449003cb20 con 0x7f449c128500 2024-09-21T01:44:58.858 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.849+0000 7f44a119d640 1 -- 172.21.15.105:0/811757463 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f449c071650 con 0x7f449c128500 2024-09-21T01:44:58.858 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.851+0000 7f44a119d640 1 -- 172.21.15.105:0/811757463 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f449c1296b0 con 0x7f449c128500 2024-09-21T01:44:58.858 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.851+0000 7f4498ff9640 1 -- 172.21.15.105:0/811757463 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f449003cc80 con 0x7f449c128500 2024-09-21T01:44:58.859 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.851+0000 7f216f7fe640 1 -- 172.21.15.105:0/477009955 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f2170004610 con 0x7f217812e8c0 2024-09-21T01:44:58.859 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.851+0000 7f217d44f640 1 -- 172.21.15.105:0/477009955 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2178112160 con 0x7f217812e8c0 2024-09-21T01:44:58.859 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.851+0000 7f4498ff9640 1 -- 172.21.15.105:0/811757463 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f4490082d10 con 0x7f449c128500 2024-09-21T01:44:58.859 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.852+0000 7f216f7fe640 1 -- 172.21.15.105:0/477009955 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f2170081280 con 0x7f217812e8c0 2024-09-21T01:44:58.860 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.852+0000 7f44a119d640 1 -- 172.21.15.105:0/811757463 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f449c127df0 con 0x7f449c128500 2024-09-21T01:44:58.860 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.856+0000 7f216f7fe640 1 -- 172.21.15.105:0/477009955 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f21700803d0 con 0x7f217812e8c0 2024-09-21T01:44:58.860 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.856+0000 7f4498ff9640 1 -- 172.21.15.105:0/811757463 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f4490083060 con 0x7f449c128500 2024-09-21T01:44:58.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.902+0000 7f95f5612640 1 -- 172.21.15.105:0/1709722901 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f95f01316f0 msgr2=0x7f95f0133ae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.902+0000 7f95f5612640 1 --2- 172.21.15.105:0/1709722901 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f95f01316f0 0x7f95f0133ae0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f95e8009be0 tx=0x7f95e802a760 comp rx=0 tx=0).stop 2024-09-21T01:44:58.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.902+0000 7f95f5612640 1 -- 172.21.15.105:0/1709722901 shutdown_connections 2024-09-21T01:44:58.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.902+0000 7f95f5612640 1 --2- 172.21.15.105:0/1709722901 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f95f01316f0 0x7f95f0133ae0 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.902+0000 7f95f5612640 1 --2- 172.21.15.105:0/1709722901 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f95f009daa0 0x7f95f01311b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.902+0000 7f95f5612640 1 -- 172.21.15.105:0/1709722901 >> 172.21.15.105:0/1709722901 conn(0x7f95f009b7c0 msgr2=0x7f95f009bbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:58.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.902+0000 7f95f5612640 1 -- 172.21.15.105:0/1709722901 shutdown_connections 2024-09-21T01:44:58.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.902+0000 7f95f5612640 1 -- 172.21.15.105:0/1709722901 wait complete. 2024-09-21T01:44:58.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.903+0000 7f95f5612640 1 Processor -- start 2024-09-21T01:44:58.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.903+0000 7f95f5612640 1 -- start start 2024-09-21T01:44:58.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.904+0000 7f95f5612640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f95f009daa0 0x7f95f012ca20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.904+0000 7f95f5612640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f95f012c430 0x7f95f012cf60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.904+0000 7f95f5612640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f95f01316f0 0x7f95f012d4a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.904+0000 7f95f5612640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f95f01307b0 con 0x7f95f012c430 2024-09-21T01:44:58.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.904+0000 7f95f5612640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f95f0129430 con 0x7f95f01316f0 2024-09-21T01:44:58.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.904+0000 7f95f5612640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f95f012ef80 con 0x7f95f009daa0 2024-09-21T01:44:58.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.904+0000 7f95ee7fc640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f95f012c430 0x7f95f012cf60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.904+0000 7f95ef7fe640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f95f01316f0 0x7f95f012d4a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.904+0000 7f95ee7fc640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f95f012c430 0x7f95f012cf60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:42106/0 (socket says 172.21.15.105:42106) 2024-09-21T01:44:58.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.904+0000 7f95ee7fc640 1 -- 172.21.15.105:0/445637072 learned_addr learned my addr 172.21.15.105:0/445637072 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:58.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.904+0000 7f95eeffd640 1 --2- 172.21.15.105:0/445637072 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f95f009daa0 0x7f95f012ca20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.904+0000 7f95ee7fc640 1 -- 172.21.15.105:0/445637072 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f95f009daa0 msgr2=0x7f95f012ca20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.904+0000 7f95ee7fc640 1 --2- 172.21.15.105:0/445637072 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f95f009daa0 0x7f95f012ca20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.904+0000 7f95ee7fc640 1 -- 172.21.15.105:0/445637072 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f95f01316f0 msgr2=0x7f95f012d4a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.904+0000 7f95ee7fc640 1 --2- 172.21.15.105:0/445637072 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f95f01316f0 0x7f95f012d4a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.904+0000 7f95ee7fc640 1 -- 172.21.15.105:0/445637072 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f95e4009560 con 0x7f95f012c430 2024-09-21T01:44:58.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.904+0000 7f95ef7fe640 1 --2- 172.21.15.105:0/445637072 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f95f01316f0 0x7f95f012d4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:58.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.904+0000 7f95eeffd640 1 --2- 172.21.15.105:0/445637072 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f95f009daa0 0x7f95f012ca20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:44:58.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.905+0000 7f95ee7fc640 1 --2- 172.21.15.105:0/445637072 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f95f012c430 0x7f95f012cf60 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f95e400e9c0 tx=0x7f95e400ee80 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:58.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.905+0000 7f95d77fe640 1 -- 172.21.15.105:0/445637072 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f95e400cde0 con 0x7f95f012c430 2024-09-21T01:44:58.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.905+0000 7f95f5612640 1 -- 172.21.15.105:0/445637072 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f95e8009840 con 0x7f95f012c430 2024-09-21T01:44:58.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.905+0000 7f95d77fe640 1 -- 172.21.15.105:0/445637072 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f95e4004020 con 0x7f95f012c430 2024-09-21T01:44:58.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.905+0000 7f95d77fe640 1 -- 172.21.15.105:0/445637072 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f95e4004430 con 0x7f95f012c430 2024-09-21T01:44:58.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.905+0000 7f95f5612640 1 -- 172.21.15.105:0/445637072 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f95f012dd50 con 0x7f95f012c430 2024-09-21T01:44:58.908 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.907+0000 7f95d77fe640 1 -- 172.21.15.105:0/445637072 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f95e40035f0 con 0x7f95f012c430 2024-09-21T01:44:58.908 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.907+0000 7f95f5612640 1 -- 172.21.15.105:0/445637072 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f95f012f140 con 0x7f95f012c430 2024-09-21T01:44:58.908 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.907+0000 7f95d77fe640 1 -- 172.21.15.105:0/445637072 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f95e4014070 con 0x7f95f012c430 2024-09-21T01:44:58.911 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.910+0000 7fbffd844640 1 -- 172.21.15.105:0/1555256304 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbff812cb20 msgr2=0x7fbff8099410 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.911 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.910+0000 7fbffd844640 1 --2- 172.21.15.105:0/1555256304 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbff812cb20 0x7fbff8099410 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7fbfec009900 tx=0x7fbfec02a530 comp rx=0 tx=0).stop 2024-09-21T01:44:58.911 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.910+0000 7fbffd844640 1 -- 172.21.15.105:0/1555256304 shutdown_connections 2024-09-21T01:44:58.911 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.910+0000 7fbffd844640 1 --2- 172.21.15.105:0/1555256304 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbff812cb20 0x7fbff8099410 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.911 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.910+0000 7fbffd844640 1 --2- 172.21.15.105:0/1555256304 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbff812c200 0x7fbff812c5e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.911 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.910+0000 7fbffd844640 1 -- 172.21.15.105:0/1555256304 >> 172.21.15.105:0/1555256304 conn(0x7fbff806c190 msgr2=0x7fbff806c5a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:58.911 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.910+0000 7fbffd844640 1 -- 172.21.15.105:0/1555256304 shutdown_connections 2024-09-21T01:44:58.911 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.910+0000 7fbffd844640 1 -- 172.21.15.105:0/1555256304 wait complete. 2024-09-21T01:44:58.913 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.911+0000 7fbffd844640 1 Processor -- start 2024-09-21T01:44:58.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.911+0000 7fbffd844640 1 -- start start 2024-09-21T01:44:58.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.912+0000 7fbffd844640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbff8099950 0x7fbff812f900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.912+0000 7fbffd844640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbff812c200 0x7fbff812fe40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.912+0000 7fbffd844640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbff812cb20 0x7fbff8130380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:44:58.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.912+0000 7fbffd844640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbff8133230 con 0x7fbff8099950 2024-09-21T01:44:58.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.912+0000 7fbffd844640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fbff8133430 con 0x7fbff812cb20 2024-09-21T01:44:58.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.912+0000 7fbffd844640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fbff8131da0 con 0x7fbff812c200 2024-09-21T01:44:58.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.912+0000 7fbff67fc640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbff812c200 0x7fbff812fe40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.912+0000 7fbff67fc640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbff812c200 0x7fbff812fe40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:55418/0 (socket says 172.21.15.105:55418) 2024-09-21T01:44:58.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.912+0000 7fbff67fc640 1 -- 172.21.15.105:0/3187738378 learned_addr learned my addr 172.21.15.105:0/3187738378 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:44:58.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.912+0000 7fbff6ffd640 1 --2- 172.21.15.105:0/3187738378 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbff8099950 0x7fbff812f900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.912+0000 7fbff77fe640 1 --2- 172.21.15.105:0/3187738378 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbff812cb20 0x7fbff8130380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:44:58.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.912+0000 7fbff67fc640 1 -- 172.21.15.105:0/3187738378 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbff812cb20 msgr2=0x7fbff8130380 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.915 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.912+0000 7fbff67fc640 1 --2- 172.21.15.105:0/3187738378 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbff812cb20 0x7fbff8130380 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.915 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.912+0000 7fbff67fc640 1 -- 172.21.15.105:0/3187738378 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbff8099950 msgr2=0x7fbff812f900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:58.915 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.912+0000 7fbff67fc640 1 --2- 172.21.15.105:0/3187738378 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbff8099950 0x7fbff812f900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:58.915 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.912+0000 7fbff67fc640 1 -- 172.21.15.105:0/3187738378 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbfec009560 con 0x7fbff812c200 2024-09-21T01:44:58.915 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.912+0000 7fbff67fc640 1 --2- 172.21.15.105:0/3187738378 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbff812c200 0x7fbff812fe40 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7fbfec006010 tx=0x7fbfec003820 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:44:58.915 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.912+0000 7fbfdffff640 1 -- 172.21.15.105:0/3187738378 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbfec003e20 con 0x7fbff812c200 2024-09-21T01:44:58.915 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.913+0000 7fbfdffff640 1 -- 172.21.15.105:0/3187738378 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fbfec03c560 con 0x7fbff812c200 2024-09-21T01:44:58.915 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.913+0000 7fbffd844640 1 -- 172.21.15.105:0/3187738378 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbff8131470 con 0x7fbff812c200 2024-09-21T01:44:58.915 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.913+0000 7fbfdffff640 1 -- 172.21.15.105:0/3187738378 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fbfec03c950 con 0x7fbff812c200 2024-09-21T01:44:58.915 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.913+0000 7fbffd844640 1 -- 172.21.15.105:0/3187738378 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbff8133e40 con 0x7fbff812c200 2024-09-21T01:44:58.916 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.914+0000 7fbffd844640 1 -- 172.21.15.105:0/3187738378 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbff8067260 con 0x7fbff812c200 2024-09-21T01:44:58.916 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.914+0000 7fbfdffff640 1 -- 172.21.15.105:0/3187738378 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7fbfec02ad50 con 0x7fbff812c200 2024-09-21T01:44:58.916 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.914+0000 7fbfdffff640 1 -- 172.21.15.105:0/3187738378 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7fbfec07da60 con 0x7fbff812c200 2024-09-21T01:44:58.916 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.914+0000 7f95d77fe640 1 -- 172.21.15.105:0/445637072 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f95e403af50 con 0x7f95f012c430 2024-09-21T01:44:58.921 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:58.921+0000 7fbfdffff640 1 -- 172.21.15.105:0/3187738378 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7fbfec047d20 con 0x7fbff812c200 2024-09-21T01:44:59.000 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.000+0000 7f217d44f640 1 -- 172.21.15.105:0/477009955 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) v1 -- 0x7f2178070600 con 0x7f217812e8c0 2024-09-21T01:44:59.001 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.000+0000 7f216f7fe640 1 -- 172.21.15.105:0/477009955 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f21700803d0 con 0x7f217812e8c0 2024-09-21T01:44:59.001 INFO:teuthology.orchestra.run.smithi105.stdout:38654705667 2024-09-21T01:44:59.003 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.002+0000 7f217d44f640 1 -- 172.21.15.105:0/477009955 >> v1:172.21.15.105:6816/3933313069 conn(0x7f2158047560 legacy=0x7f2158049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:59.003 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.002+0000 7f217d44f640 1 -- 172.21.15.105:0/477009955 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f217812e8c0 msgr2=0x7f2178120b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:59.003 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.002+0000 7f217d44f640 1 --2- 172.21.15.105:0/477009955 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f217812e8c0 0x7f2178120b50 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f2170009b30 tx=0x7f2170003c10 comp rx=0 tx=0).stop 2024-09-21T01:44:59.003 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.002+0000 7f217d44f640 1 -- 172.21.15.105:0/477009955 shutdown_connections 2024-09-21T01:44:59.003 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.002+0000 7f217d44f640 1 --2- 172.21.15.105:0/477009955 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f217812e8c0 0x7f2178120b50 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:59.003 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.002+0000 7f217d44f640 1 --2- 172.21.15.105:0/477009955 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f217812bf90 0x7f2178120610 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:59.003 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.002+0000 7f217d44f640 1 --2- 172.21.15.105:0/477009955 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2178128f80 0x7f21781200d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:59.003 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.002+0000 7f217d44f640 1 -- 172.21.15.105:0/477009955 >> 172.21.15.105:0/477009955 conn(0x7f2178099740 msgr2=0x7f217812d010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:59.003 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.002+0000 7f217d44f640 1 -- 172.21.15.105:0/477009955 shutdown_connections 2024-09-21T01:44:59.003 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.002+0000 7f217d44f640 1 -- 172.21.15.105:0/477009955 wait complete. 2024-09-21T01:44:59.013 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705668 got 38654705667 for osd.3 2024-09-21T01:44:59.014 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.014+0000 7f44a119d640 1 -- 172.21.15.105:0/811757463 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) v1 -- 0x7f449c04e210 con 0x7f449c128500 2024-09-21T01:44:59.015 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.014+0000 7f4498ff9640 1 -- 172.21.15.105:0/811757463 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f4490049030 con 0x7f449c128500 2024-09-21T01:44:59.015 INFO:teuthology.orchestra.run.smithi105.stdout:38654705667 2024-09-21T01:44:59.017 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.016+0000 7f44a119d640 1 -- 172.21.15.105:0/811757463 >> v1:172.21.15.105:6816/3933313069 conn(0x7f4488047560 legacy=0x7f4488049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:59.017 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.016+0000 7f44a119d640 1 -- 172.21.15.105:0/811757463 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f449c128500 msgr2=0x7f449c070bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:59.017 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.016+0000 7f44a119d640 1 --2- 172.21.15.105:0/811757463 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f449c128500 0x7f449c070bd0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f4490000c00 tx=0x7f4490003c10 comp rx=0 tx=0).stop 2024-09-21T01:44:59.017 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.016+0000 7f44a119d640 1 -- 172.21.15.105:0/811757463 shutdown_connections 2024-09-21T01:44:59.017 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.016+0000 7f44a119d640 1 --2- 172.21.15.105:0/811757463 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f449c12f3e0 0x7f449c071110 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:59.017 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.016+0000 7f44a119d640 1 --2- 172.21.15.105:0/811757463 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f449c128500 0x7f449c070bd0 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:59.017 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.016+0000 7f44a119d640 1 --2- 172.21.15.105:0/811757463 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f449c127a10 0x7f449c070690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:59.017 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.016+0000 7f44a119d640 1 -- 172.21.15.105:0/811757463 >> 172.21.15.105:0/811757463 conn(0x7f449c09c710 msgr2=0x7f449c1313d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:59.017 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.016+0000 7f44a119d640 1 -- 172.21.15.105:0/811757463 shutdown_connections 2024-09-21T01:44:59.017 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.016+0000 7f44a119d640 1 -- 172.21.15.105:0/811757463 wait complete. 2024-09-21T01:44:59.027 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705668 got 38654705667 for osd.2 2024-09-21T01:44:59.061 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.061+0000 7f95f5612640 1 -- 172.21.15.105:0/445637072 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) v1 -- 0x7f95f012fb00 con 0x7f95f012c430 2024-09-21T01:44:59.062 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.061+0000 7f95d77fe640 1 -- 172.21.15.105:0/445637072 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f95e403e570 con 0x7f95f012c430 2024-09-21T01:44:59.062 INFO:teuthology.orchestra.run.smithi105.stdout:38654705667 2024-09-21T01:44:59.064 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.063+0000 7f95f5612640 1 -- 172.21.15.105:0/445637072 >> v1:172.21.15.105:6816/3933313069 conn(0x7f95dc047590 legacy=0x7f95dc049a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:59.064 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.063+0000 7f95f5612640 1 -- 172.21.15.105:0/445637072 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f95f012c430 msgr2=0x7f95f012cf60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:59.064 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.063+0000 7f95f5612640 1 --2- 172.21.15.105:0/445637072 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f95f012c430 0x7f95f012cf60 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f95e400e9c0 tx=0x7f95e400ee80 comp rx=0 tx=0).stop 2024-09-21T01:44:59.064 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.063+0000 7f95f5612640 1 -- 172.21.15.105:0/445637072 shutdown_connections 2024-09-21T01:44:59.064 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.063+0000 7f95f5612640 1 --2- 172.21.15.105:0/445637072 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f95f01316f0 0x7f95f012d4a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:59.064 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.063+0000 7f95f5612640 1 --2- 172.21.15.105:0/445637072 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f95f012c430 0x7f95f012cf60 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:59.064 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.063+0000 7f95f5612640 1 --2- 172.21.15.105:0/445637072 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f95f009daa0 0x7f95f012ca20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:59.064 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.063+0000 7f95f5612640 1 -- 172.21.15.105:0/445637072 >> 172.21.15.105:0/445637072 conn(0x7f95f009b7c0 msgr2=0x7f95f0130c70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:59.064 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.063+0000 7f95f5612640 1 -- 172.21.15.105:0/445637072 shutdown_connections 2024-09-21T01:44:59.064 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.063+0000 7f95f5612640 1 -- 172.21.15.105:0/445637072 wait complete. 2024-09-21T01:44:59.065 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.064+0000 7fbffd844640 1 -- 172.21.15.105:0/3187738378 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) v1 -- 0x7fbff81340f0 con 0x7fbff812c200 2024-09-21T01:44:59.065 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.064+0000 7fbfdffff640 1 -- 172.21.15.105:0/3187738378 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7fbfec035070 con 0x7fbff812c200 2024-09-21T01:44:59.065 INFO:teuthology.orchestra.run.smithi105.stdout:38654705668 2024-09-21T01:44:59.067 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.066+0000 7fbffd844640 1 -- 172.21.15.105:0/3187738378 >> v1:172.21.15.105:6816/3933313069 conn(0x7fbfe4047560 legacy=0x7fbfe4049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:59.067 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.066+0000 7fbffd844640 1 -- 172.21.15.105:0/3187738378 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbff812c200 msgr2=0x7fbff812fe40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:44:59.067 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.066+0000 7fbffd844640 1 --2- 172.21.15.105:0/3187738378 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbff812c200 0x7fbff812fe40 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7fbfec006010 tx=0x7fbfec003820 comp rx=0 tx=0).stop 2024-09-21T01:44:59.067 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.066+0000 7fbffd844640 1 -- 172.21.15.105:0/3187738378 shutdown_connections 2024-09-21T01:44:59.067 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.066+0000 7fbffd844640 1 --2- 172.21.15.105:0/3187738378 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbff812cb20 0x7fbff8130380 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:59.067 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.066+0000 7fbffd844640 1 --2- 172.21.15.105:0/3187738378 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbff812c200 0x7fbff812fe40 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:59.067 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.066+0000 7fbffd844640 1 --2- 172.21.15.105:0/3187738378 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbff8099950 0x7fbff812f900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:44:59.067 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.066+0000 7fbffd844640 1 -- 172.21.15.105:0/3187738378 >> 172.21.15.105:0/3187738378 conn(0x7fbff806c190 msgr2=0x7fbff809dee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:44:59.068 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.066+0000 7fbffd844640 1 -- 172.21.15.105:0/3187738378 shutdown_connections 2024-09-21T01:44:59.068 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:44:59.066+0000 7fbffd844640 1 -- 172.21.15.105:0/3187738378 wait complete. 2024-09-21T01:44:59.075 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705668 got 38654705667 for osd.1 2024-09-21T01:44:59.077 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705669 got 38654705668 for osd.0 2024-09-21T01:45:00.015 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T01:45:00.028 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T01:45:00.076 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T01:45:00.078 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T01:45:00.131 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.131+0000 7fd1e614c640 1 -- 172.21.15.105:0/2717311198 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd1e0128b00 msgr2=0x7fd1e0128ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:00.132 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.131+0000 7fd1e614c640 1 --2- 172.21.15.105:0/2717311198 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd1e0128b00 0x7fd1e0128ee0 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fd1d4009830 tx=0x7fd1d402a3f0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.132 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.131+0000 7fd1e614c640 1 -- 172.21.15.105:0/2717311198 shutdown_connections 2024-09-21T01:45:00.132 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.131+0000 7fd1e614c640 1 --2- 172.21.15.105:0/2717311198 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd1e0128b00 0x7fd1e0128ee0 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.132 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.131+0000 7fd1e614c640 1 --2- 172.21.15.105:0/2717311198 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd1e01259d0 0x7fd1e0125db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.132 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.131+0000 7fd1e614c640 1 -- 172.21.15.105:0/2717311198 >> 172.21.15.105:0/2717311198 conn(0x7fd1e0099740 msgr2=0x7fd1e0099b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:00.132 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.131+0000 7fd1e614c640 1 -- 172.21.15.105:0/2717311198 shutdown_connections 2024-09-21T01:45:00.132 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.131+0000 7fd1e614c640 1 -- 172.21.15.105:0/2717311198 wait complete. 2024-09-21T01:45:00.135 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.132+0000 7fd1e614c640 1 Processor -- start 2024-09-21T01:45:00.135 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.132+0000 7fd1e614c640 1 -- start start 2024-09-21T01:45:00.135 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.133+0000 7fd1e614c640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd1e01259d0 0x7fd1e012da40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:00.135 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.133+0000 7fd1e614c640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd1e0128b00 0x7fd1e012df80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:00.135 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.133+0000 7fd1e614c640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd1e0129420 0x7fd1e012e4c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:00.135 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.133+0000 7fd1e614c640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd1e011fac0 con 0x7fd1e01259d0 2024-09-21T01:45:00.135 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.133+0000 7fd1e614c640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fd1e012fef0 con 0x7fd1e0129420 2024-09-21T01:45:00.135 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.133+0000 7fd1e614c640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fd1e0130b30 con 0x7fd1e0128b00 2024-09-21T01:45:00.135 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.133+0000 7fd1dffff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd1e0129420 0x7fd1e012e4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:00.135 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.133+0000 7fd1dffff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd1e0129420 0x7fd1e012e4c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:58868/0 (socket says 172.21.15.105:58868) 2024-09-21T01:45:00.135 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.133+0000 7fd1dffff640 1 -- 172.21.15.105:0/2055792935 learned_addr learned my addr 172.21.15.105:0/2055792935 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:45:00.136 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.133+0000 7fd1deffd640 1 --2- 172.21.15.105:0/2055792935 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd1e0128b00 0x7fd1e012df80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:00.136 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.133+0000 7fd1dffff640 1 -- 172.21.15.105:0/2055792935 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd1e0128b00 msgr2=0x7fd1e012df80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:00.136 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.133+0000 7fd1df7fe640 1 --2- 172.21.15.105:0/2055792935 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd1e01259d0 0x7fd1e012da40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:00.136 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.133+0000 7fd1dffff640 1 --2- 172.21.15.105:0/2055792935 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd1e0128b00 0x7fd1e012df80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.136 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.133+0000 7fd1dffff640 1 -- 172.21.15.105:0/2055792935 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd1e01259d0 msgr2=0x7fd1e012da40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:00.136 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.134+0000 7fd1dffff640 1 --2- 172.21.15.105:0/2055792935 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd1e01259d0 0x7fd1e012da40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.136 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.134+0000 7fd1dffff640 1 -- 172.21.15.105:0/2055792935 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd1d4009490 con 0x7fd1e0129420 2024-09-21T01:45:00.136 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.134+0000 7fd1deffd640 1 --2- 172.21.15.105:0/2055792935 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd1e0128b00 0x7fd1e012df80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:45:00.136 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.134+0000 7fd1df7fe640 1 --2- 172.21.15.105:0/2055792935 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd1e01259d0 0x7fd1e012da40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T01:45:00.136 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.134+0000 7fd1dffff640 1 --2- 172.21.15.105:0/2055792935 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd1e0129420 0x7fd1e012e4c0 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fd1d800c760 tx=0x7fd1d800cc20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:45:00.136 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.134+0000 7fd1c7fff640 1 -- 172.21.15.105:0/2055792935 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd1d8003da0 con 0x7fd1e0129420 2024-09-21T01:45:00.136 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.134+0000 7fd1c7fff640 1 -- 172.21.15.105:0/2055792935 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fd1d800ce70 con 0x7fd1e0129420 2024-09-21T01:45:00.136 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.134+0000 7fd1c7fff640 1 -- 172.21.15.105:0/2055792935 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fd1d80052d0 con 0x7fd1e0129420 2024-09-21T01:45:00.136 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.134+0000 7fd1e614c640 1 -- 172.21.15.105:0/2055792935 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd1e012fac0 con 0x7fd1e0129420 2024-09-21T01:45:00.137 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.135+0000 7fd1e614c640 1 -- 172.21.15.105:0/2055792935 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd1e012f6a0 con 0x7fd1e0129420 2024-09-21T01:45:00.137 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.135+0000 7fd1c7fff640 1 -- 172.21.15.105:0/2055792935 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7fd1d8017070 con 0x7fd1e0129420 2024-09-21T01:45:00.137 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.136+0000 7fd1c7fff640 1 -- 172.21.15.105:0/2055792935 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7fd1d8063dc0 con 0x7fd1e0129420 2024-09-21T01:45:00.137 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.136+0000 7fd1e614c640 1 -- 172.21.15.105:0/2055792935 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd1e012fc50 con 0x7fd1e0129420 2024-09-21T01:45:00.141 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.140+0000 7fd1c7fff640 1 -- 172.21.15.105:0/2055792935 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7fd1d803d520 con 0x7fd1e0129420 2024-09-21T01:45:00.169 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.169+0000 7f3b3523a640 1 -- 172.21.15.105:0/1308739391 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3b3009daa0 msgr2=0x7f3b301288b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:00.169 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.169+0000 7f3b3523a640 1 --2- 172.21.15.105:0/1308739391 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3b3009daa0 0x7f3b301288b0 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f3b24005fd0 tx=0x7f3b2402a4b0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.169 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.169+0000 7f3b3523a640 1 -- 172.21.15.105:0/1308739391 shutdown_connections 2024-09-21T01:45:00.169 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.169+0000 7f3b3523a640 1 --2- 172.21.15.105:0/1308739391 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3b3009daa0 0x7f3b301288b0 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.170 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.169+0000 7f3b3523a640 1 --2- 172.21.15.105:0/1308739391 >> v2:172.21.15.105:3300/0 conn(0x7f3b3012c430 0x7f3b3012c810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.170 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.169+0000 7f3b3523a640 1 -- 172.21.15.105:0/1308739391 >> 172.21.15.105:0/1308739391 conn(0x7f3b3009b7c0 msgr2=0x7f3b3009bbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:00.170 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.169+0000 7f3b3523a640 1 -- 172.21.15.105:0/1308739391 shutdown_connections 2024-09-21T01:45:00.170 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.169+0000 7f3b3523a640 1 -- 172.21.15.105:0/1308739391 wait complete. 2024-09-21T01:45:00.172 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.170+0000 7f3b3523a640 1 Processor -- start 2024-09-21T01:45:00.172 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.170+0000 7f3b3523a640 1 -- start start 2024-09-21T01:45:00.172 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.170+0000 7f3b3523a640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3b3009daa0 0x7f3b3012ff20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:00.172 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.170+0000 7f3b3523a640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3b30128df0 0x7f3b30130460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:00.172 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.170+0000 7f3b3523a640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3b3012c430 0x7f3b301309a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:00.172 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.170+0000 7f3b3523a640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3b30134310 con 0x7f3b3009daa0 2024-09-21T01:45:00.172 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.170+0000 7f3b3523a640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f3b30132010 con 0x7f3b30128df0 2024-09-21T01:45:00.172 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.170+0000 7f3b3523a640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f3b30132860 con 0x7f3b3012c430 2024-09-21T01:45:00.172 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.171+0000 7f3b2e59c640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3b30128df0 0x7f3b30130460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:00.173 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.171+0000 7f3b2ed9d640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3b3009daa0 0x7f3b3012ff20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:00.173 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.171+0000 7f3b2ed9d640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3b3009daa0 0x7f3b3012ff20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:42152/0 (socket says 172.21.15.105:42152) 2024-09-21T01:45:00.173 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.171+0000 7f3b2e59c640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3b30128df0 0x7f3b30130460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:58888/0 (socket says 172.21.15.105:58888) 2024-09-21T01:45:00.173 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.171+0000 7f3b2ed9d640 1 -- 172.21.15.105:0/3542990137 learned_addr learned my addr 172.21.15.105:0/3542990137 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:45:00.173 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.171+0000 7f3b2f59e640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3b3012c430 0x7f3b301309a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:00.173 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.171+0000 7f3b2ed9d640 1 -- 172.21.15.105:0/3542990137 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3b3012c430 msgr2=0x7f3b301309a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:00.173 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.171+0000 7f3b2ed9d640 1 --2- 172.21.15.105:0/3542990137 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3b3012c430 0x7f3b301309a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.173 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.171+0000 7f3b2ed9d640 1 -- 172.21.15.105:0/3542990137 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3b30128df0 msgr2=0x7f3b30130460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:00.173 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.171+0000 7f3b2ed9d640 1 --2- 172.21.15.105:0/3542990137 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3b30128df0 0x7f3b30130460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.173 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.171+0000 7f3b2ed9d640 1 -- 172.21.15.105:0/3542990137 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3b20009490 con 0x7f3b3009daa0 2024-09-21T01:45:00.173 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.171+0000 7f3b2f59e640 1 --2- 172.21.15.105:0/3542990137 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3b3012c430 0x7f3b301309a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:45:00.173 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.171+0000 7f3b2e59c640 1 --2- 172.21.15.105:0/3542990137 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3b30128df0 0x7f3b30130460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:45:00.173 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.171+0000 7f3b2ed9d640 1 --2- 172.21.15.105:0/3542990137 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3b3009daa0 0x7f3b3012ff20 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f3b2000ea60 tx=0x7f3b2000ef20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:45:00.173 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.171+0000 7f3b177fe640 1 -- 172.21.15.105:0/3542990137 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b2000cde0 con 0x7f3b3009daa0 2024-09-21T01:45:00.174 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.171+0000 7f3b3523a640 1 -- 172.21.15.105:0/3542990137 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3b24009560 con 0x7f3b3009daa0 2024-09-21T01:45:00.174 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.171+0000 7f3b177fe640 1 -- 172.21.15.105:0/3542990137 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f3b20004020 con 0x7f3b3009daa0 2024-09-21T01:45:00.174 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.171+0000 7f3b177fe640 1 -- 172.21.15.105:0/3542990137 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f3b20004410 con 0x7f3b3009daa0 2024-09-21T01:45:00.174 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.171+0000 7f3b3523a640 1 -- 172.21.15.105:0/3542990137 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3b30130ee0 con 0x7f3b3009daa0 2024-09-21T01:45:00.174 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.172+0000 7f3b3523a640 1 -- 172.21.15.105:0/3542990137 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3b301288b0 con 0x7f3b3009daa0 2024-09-21T01:45:00.174 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.173+0000 7f3b177fe640 1 -- 172.21.15.105:0/3542990137 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f3b200035f0 con 0x7f3b3009daa0 2024-09-21T01:45:00.174 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.173+0000 7f3b177fe640 1 -- 172.21.15.105:0/3542990137 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f3b20014070 con 0x7f3b3009daa0 2024-09-21T01:45:00.180 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.180+0000 7f3b177fe640 1 -- 172.21.15.105:0/3542990137 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f3b2003af50 con 0x7f3b3009daa0 2024-09-21T01:45:00.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.212+0000 7f2f0a1ed640 1 -- 172.21.15.105:0/225884393 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2f04128df0 msgr2=0x7f2f04129280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:00.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.212+0000 7f2f0a1ed640 1 --2- 172.21.15.105:0/225884393 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2f04128df0 0x7f2f04129280 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f2efc005fd0 tx=0x7f2efc02a8f0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.212+0000 7f2f0a1ed640 1 -- 172.21.15.105:0/225884393 shutdown_connections 2024-09-21T01:45:00.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.212+0000 7f2f0a1ed640 1 --2- 172.21.15.105:0/225884393 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2f04128df0 0x7f2f04129280 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.212+0000 7f2f0a1ed640 1 --2- 172.21.15.105:0/225884393 >> v2:172.21.15.105:3300/0 conn(0x7f2f0409daa0 0x7f2f041288b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.212+0000 7f2f0a1ed640 1 --2- 172.21.15.105:0/225884393 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2f0412c430 0x7f2f0412c810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.212+0000 7f2f0a1ed640 1 -- 172.21.15.105:0/225884393 >> 172.21.15.105:0/225884393 conn(0x7f2f0409b7c0 msgr2=0x7f2f0409bbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:00.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.212+0000 7f2f0a1ed640 1 -- 172.21.15.105:0/225884393 shutdown_connections 2024-09-21T01:45:00.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.212+0000 7f2f0a1ed640 1 -- 172.21.15.105:0/225884393 wait complete. 2024-09-21T01:45:00.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.213+0000 7f2f0a1ed640 1 Processor -- start 2024-09-21T01:45:00.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.213+0000 7f2f0a1ed640 1 -- start start 2024-09-21T01:45:00.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.214+0000 7f2f0a1ed640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2f0409daa0 0x7f2f041c20b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:00.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.214+0000 7f2f0a1ed640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2f04128df0 0x7f2f041c25f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:00.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.214+0000 7f2f0a1ed640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2f0412c430 0x7f2f0412fb00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:00.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.214+0000 7f2f0a1ed640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2f04131f30 con 0x7f2f0409daa0 2024-09-21T01:45:00.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.214+0000 7f2f0a1ed640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2f041317a0 con 0x7f2f0412c430 2024-09-21T01:45:00.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.214+0000 7f2f0a1ed640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f2f04132ae0 con 0x7f2f04128df0 2024-09-21T01:45:00.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.214+0000 7f2f037fe640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2f0409daa0 0x7f2f041c20b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:00.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.214+0000 7f2f037fe640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2f0409daa0 0x7f2f041c20b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:42182/0 (socket says 172.21.15.105:42182) 2024-09-21T01:45:00.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.214+0000 7f2f03fff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2f0412c430 0x7f2f0412fb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:00.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.214+0000 7f2f037fe640 1 -- 172.21.15.105:0/4285392282 learned_addr learned my addr 172.21.15.105:0/4285392282 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:45:00.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.214+0000 7f2f02ffd640 1 --2- 172.21.15.105:0/4285392282 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2f04128df0 0x7f2f041c25f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:00.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.214+0000 7f2f037fe640 1 -- 172.21.15.105:0/4285392282 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2f04128df0 msgr2=0x7f2f041c25f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:00.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.214+0000 7f2f037fe640 1 --2- 172.21.15.105:0/4285392282 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2f04128df0 0x7f2f041c25f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.214+0000 7f2f037fe640 1 -- 172.21.15.105:0/4285392282 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2f0412c430 msgr2=0x7f2f0412fb00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:00.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.214+0000 7f2f037fe640 1 --2- 172.21.15.105:0/4285392282 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2f0412c430 0x7f2f0412fb00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.215+0000 7f2f037fe640 1 -- 172.21.15.105:0/4285392282 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2efc009900 con 0x7f2f0409daa0 2024-09-21T01:45:00.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.215+0000 7f2f03fff640 1 --2- 172.21.15.105:0/4285392282 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2f0412c430 0x7f2f0412fb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:45:00.217 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.215+0000 7f2f037fe640 1 --2- 172.21.15.105:0/4285392282 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2f0409daa0 0x7f2f041c20b0 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f2ef400b320 tx=0x7f2ef400b7e0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:45:00.217 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.215+0000 7f2eebfff640 1 -- 172.21.15.105:0/4285392282 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2ef4019070 con 0x7f2f0409daa0 2024-09-21T01:45:00.217 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.215+0000 7f2eebfff640 1 -- 172.21.15.105:0/4285392282 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f2ef400a520 con 0x7f2f0409daa0 2024-09-21T01:45:00.217 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.215+0000 7f2f0a1ed640 1 -- 172.21.15.105:0/4285392282 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2f04115260 con 0x7f2f0409daa0 2024-09-21T01:45:00.217 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.215+0000 7f2eebfff640 1 -- 172.21.15.105:0/4285392282 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2ef4014c70 con 0x7f2f0409daa0 2024-09-21T01:45:00.217 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.215+0000 7f2f0a1ed640 1 -- 172.21.15.105:0/4285392282 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2f041c2b30 con 0x7f2f0409daa0 2024-09-21T01:45:00.217 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.217+0000 7f2f0a1ed640 1 -- 172.21.15.105:0/4285392282 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2f04129280 con 0x7f2f0409daa0 2024-09-21T01:45:00.217 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.217+0000 7f2eebfff640 1 -- 172.21.15.105:0/4285392282 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f2ef400a8f0 con 0x7f2f0409daa0 2024-09-21T01:45:00.218 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.217+0000 7f2eebfff640 1 -- 172.21.15.105:0/4285392282 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f2ef4064a50 con 0x7f2f0409daa0 2024-09-21T01:45:00.221 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.221+0000 7f2eebfff640 1 -- 172.21.15.105:0/4285392282 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f2ef40426e0 con 0x7f2f0409daa0 2024-09-21T01:45:00.244 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.243+0000 7fa491176640 1 -- 172.21.15.105:0/2643515613 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa48c128df0 msgr2=0x7fa48c129280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:00.244 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.243+0000 7fa491176640 1 --2- 172.21.15.105:0/2643515613 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa48c128df0 0x7fa48c129280 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7fa484005fd0 tx=0x7fa48402a6f0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.244 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.243+0000 7fa491176640 1 -- 172.21.15.105:0/2643515613 shutdown_connections 2024-09-21T01:45:00.244 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.243+0000 7fa491176640 1 --2- 172.21.15.105:0/2643515613 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa48c128df0 0x7fa48c129280 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.244 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.243+0000 7fa491176640 1 --2- 172.21.15.105:0/2643515613 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa48c09daa0 0x7fa48c1288b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.244 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.243+0000 7fa491176640 1 --2- 172.21.15.105:0/2643515613 >> v2:172.21.15.105:3300/0 conn(0x7fa48c12c430 0x7fa48c12c810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.244 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.243+0000 7fa491176640 1 -- 172.21.15.105:0/2643515613 >> 172.21.15.105:0/2643515613 conn(0x7fa48c09b7c0 msgr2=0x7fa48c09bbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:00.244 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.243+0000 7fa491176640 1 -- 172.21.15.105:0/2643515613 shutdown_connections 2024-09-21T01:45:00.244 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.243+0000 7fa491176640 1 -- 172.21.15.105:0/2643515613 wait complete. 2024-09-21T01:45:00.245 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.245+0000 7fa491176640 1 Processor -- start 2024-09-21T01:45:00.245 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.245+0000 7fa491176640 1 -- start start 2024-09-21T01:45:00.247 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.245+0000 7fa491176640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa48c09daa0 0x7fa48c12fb60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:00.247 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.245+0000 7fa491176640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa48c128df0 0x7fa48c1300a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:00.247 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.245+0000 7fa491176640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa48c12c430 0x7fa48c1305e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:00.247 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.245+0000 7fa491176640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa48c114ef0 con 0x7fa48c09daa0 2024-09-21T01:45:00.247 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.245+0000 7fa491176640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fa48c131de0 con 0x7fa48c12c430 2024-09-21T01:45:00.248 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.245+0000 7fa491176640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fa48c122db0 con 0x7fa48c128df0 2024-09-21T01:45:00.248 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.246+0000 7fa48ad9d640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa48c09daa0 0x7fa48c12fb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:00.248 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.246+0000 7fa48a59c640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa48c128df0 0x7fa48c1300a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:00.248 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.246+0000 7fa48a59c640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa48c128df0 0x7fa48c1300a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:55494/0 (socket says 172.21.15.105:55494) 2024-09-21T01:45:00.248 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.246+0000 7fa48a59c640 1 -- 172.21.15.105:0/275763910 learned_addr learned my addr 172.21.15.105:0/275763910 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:45:00.248 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.246+0000 7fa48b59e640 1 --2- 172.21.15.105:0/275763910 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa48c12c430 0x7fa48c1305e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:00.248 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.246+0000 7fa48a59c640 1 -- 172.21.15.105:0/275763910 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa48c12c430 msgr2=0x7fa48c1305e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:00.248 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.246+0000 7fa48a59c640 1 --2- 172.21.15.105:0/275763910 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa48c12c430 0x7fa48c1305e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.248 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.246+0000 7fa48a59c640 1 -- 172.21.15.105:0/275763910 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa48c09daa0 msgr2=0x7fa48c12fb60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:00.248 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.246+0000 7fa48a59c640 1 --2- 172.21.15.105:0/275763910 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa48c09daa0 0x7fa48c12fb60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.248 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.246+0000 7fa48a59c640 1 -- 172.21.15.105:0/275763910 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa480009560 con 0x7fa48c128df0 2024-09-21T01:45:00.249 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.246+0000 7fa48ad9d640 1 --2- 172.21.15.105:0/275763910 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa48c09daa0 0x7fa48c12fb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T01:45:00.249 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.246+0000 7fa48a59c640 1 --2- 172.21.15.105:0/275763910 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa48c128df0 0x7fa48c1300a0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7fa48000e9f0 tx=0x7fa48000eeb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:45:00.249 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.246+0000 7fa4737fe640 1 -- 172.21.15.105:0/275763910 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa48000cde0 con 0x7fa48c128df0 2024-09-21T01:45:00.249 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.247+0000 7fa491176640 1 -- 172.21.15.105:0/275763910 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa47c009490 con 0x7fa48c128df0 2024-09-21T01:45:00.249 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.247+0000 7fa4737fe640 1 -- 172.21.15.105:0/275763910 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fa480004020 con 0x7fa48c128df0 2024-09-21T01:45:00.250 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.247+0000 7fa4737fe640 1 -- 172.21.15.105:0/275763910 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fa480004410 con 0x7fa48c128df0 2024-09-21T01:45:00.250 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.247+0000 7fa491176640 1 -- 172.21.15.105:0/275763910 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa484009840 con 0x7fa48c128df0 2024-09-21T01:45:00.250 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.248+0000 7fa491176640 1 -- 172.21.15.105:0/275763910 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa48c129280 con 0x7fa48c128df0 2024-09-21T01:45:00.250 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.248+0000 7fa4737fe640 1 -- 172.21.15.105:0/275763910 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7fa4800035f0 con 0x7fa48c128df0 2024-09-21T01:45:00.250 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.248+0000 7fa4737fe640 1 -- 172.21.15.105:0/275763910 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7fa480014070 con 0x7fa48c128df0 2024-09-21T01:45:00.252 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.252+0000 7fa4737fe640 1 -- 172.21.15.105:0/275763910 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7fa48003af60 con 0x7fa48c128df0 2024-09-21T01:45:00.327 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.326+0000 7f3b3523a640 1 -- 172.21.15.105:0/3542990137 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) v1 -- 0x7f3b30124950 con 0x7f3b3009daa0 2024-09-21T01:45:00.327 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.327+0000 7f3b177fe640 1 -- 172.21.15.105:0/3542990137 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f3b2003e570 con 0x7f3b3009daa0 2024-09-21T01:45:00.327 INFO:teuthology.orchestra.run.smithi105.stdout:38654705667 2024-09-21T01:45:00.329 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.328+0000 7f3b3523a640 1 -- 172.21.15.105:0/3542990137 >> v1:172.21.15.105:6816/3933313069 conn(0x7f3b1c047560 legacy=0x7f3b1c049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:00.329 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.328+0000 7f3b3523a640 1 -- 172.21.15.105:0/3542990137 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3b3009daa0 msgr2=0x7f3b3012ff20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:00.329 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.328+0000 7f3b3523a640 1 --2- 172.21.15.105:0/3542990137 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3b3009daa0 0x7f3b3012ff20 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f3b2000ea60 tx=0x7f3b2000ef20 comp rx=0 tx=0).stop 2024-09-21T01:45:00.329 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.328+0000 7f3b3523a640 1 -- 172.21.15.105:0/3542990137 shutdown_connections 2024-09-21T01:45:00.329 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.328+0000 7f3b3523a640 1 --2- 172.21.15.105:0/3542990137 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3b3012c430 0x7f3b301309a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.329 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.328+0000 7f3b3523a640 1 --2- 172.21.15.105:0/3542990137 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3b30128df0 0x7f3b30130460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.329 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.328+0000 7f3b3523a640 1 --2- 172.21.15.105:0/3542990137 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3b3009daa0 0x7f3b3012ff20 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.329 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.328+0000 7f3b3523a640 1 -- 172.21.15.105:0/3542990137 >> 172.21.15.105:0/3542990137 conn(0x7f3b3009b7c0 msgr2=0x7f3b30099dd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:00.329 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.328+0000 7f3b3523a640 1 -- 172.21.15.105:0/3542990137 shutdown_connections 2024-09-21T01:45:00.329 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.328+0000 7f3b3523a640 1 -- 172.21.15.105:0/3542990137 wait complete. 2024-09-21T01:45:00.340 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705668 got 38654705667 for osd.2 2024-09-21T01:45:00.367 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.366+0000 7f2f0a1ed640 1 -- 172.21.15.105:0/4285392282 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) v1 -- 0x7f2f04115240 con 0x7f2f0409daa0 2024-09-21T01:45:00.367 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.367+0000 7f2eebfff640 1 -- 172.21.15.105:0/4285392282 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f2ef4022030 con 0x7f2f0409daa0 2024-09-21T01:45:00.367 INFO:teuthology.orchestra.run.smithi105.stdout:38654705667 2024-09-21T01:45:00.369 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.368+0000 7f2f0a1ed640 1 -- 172.21.15.105:0/4285392282 >> v1:172.21.15.105:6816/3933313069 conn(0x7f2ef0047560 legacy=0x7f2ef0049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:00.369 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.368+0000 7f2f0a1ed640 1 -- 172.21.15.105:0/4285392282 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2f0409daa0 msgr2=0x7f2f041c20b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:00.369 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.368+0000 7f2f0a1ed640 1 --2- 172.21.15.105:0/4285392282 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2f0409daa0 0x7f2f041c20b0 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7f2ef400b320 tx=0x7f2ef400b7e0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.369 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.368+0000 7f2f0a1ed640 1 -- 172.21.15.105:0/4285392282 shutdown_connections 2024-09-21T01:45:00.369 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.368+0000 7f2f0a1ed640 1 --2- 172.21.15.105:0/4285392282 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2f0412c430 0x7f2f0412fb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.369 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.368+0000 7f2f0a1ed640 1 --2- 172.21.15.105:0/4285392282 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2f04128df0 0x7f2f041c25f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.369 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.368+0000 7f2f0a1ed640 1 --2- 172.21.15.105:0/4285392282 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2f0409daa0 0x7f2f041c20b0 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.370 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.368+0000 7f2f0a1ed640 1 -- 172.21.15.105:0/4285392282 >> 172.21.15.105:0/4285392282 conn(0x7f2f0409b7c0 msgr2=0x7f2f0412a790 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:00.370 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.369+0000 7f2f0a1ed640 1 -- 172.21.15.105:0/4285392282 shutdown_connections 2024-09-21T01:45:00.370 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.369+0000 7f2f0a1ed640 1 -- 172.21.15.105:0/4285392282 wait complete. 2024-09-21T01:45:00.370 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.370+0000 7fd1e614c640 1 -- 172.21.15.105:0/2055792935 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) v1 -- 0x7fd1e0125db0 con 0x7fd1e0129420 2024-09-21T01:45:00.370 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.370+0000 7fd1c7fff640 1 -- 172.21.15.105:0/2055792935 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7fd1d801f030 con 0x7fd1e0129420 2024-09-21T01:45:00.371 INFO:teuthology.orchestra.run.smithi105.stdout:38654705667 2024-09-21T01:45:00.372 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.372+0000 7fd1e614c640 1 -- 172.21.15.105:0/2055792935 >> v1:172.21.15.105:6816/3933313069 conn(0x7fd1cc047560 legacy=0x7fd1cc049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:00.372 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.372+0000 7fd1e614c640 1 -- 172.21.15.105:0/2055792935 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd1e0129420 msgr2=0x7fd1e012e4c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:00.372 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.372+0000 7fd1e614c640 1 --2- 172.21.15.105:0/2055792935 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd1e0129420 0x7fd1e012e4c0 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fd1d800c760 tx=0x7fd1d800cc20 comp rx=0 tx=0).stop 2024-09-21T01:45:00.373 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.372+0000 7fd1e614c640 1 -- 172.21.15.105:0/2055792935 shutdown_connections 2024-09-21T01:45:00.373 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.372+0000 7fd1e614c640 1 --2- 172.21.15.105:0/2055792935 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd1e0129420 0x7fd1e012e4c0 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.373 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.372+0000 7fd1e614c640 1 --2- 172.21.15.105:0/2055792935 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd1e0128b00 0x7fd1e012df80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.373 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.372+0000 7fd1e614c640 1 --2- 172.21.15.105:0/2055792935 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd1e01259d0 0x7fd1e012da40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.373 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.372+0000 7fd1e614c640 1 -- 172.21.15.105:0/2055792935 >> 172.21.15.105:0/2055792935 conn(0x7fd1e0099740 msgr2=0x7fd1e011ff50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:00.373 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.372+0000 7fd1e614c640 1 -- 172.21.15.105:0/2055792935 shutdown_connections 2024-09-21T01:45:00.373 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.372+0000 7fd1e614c640 1 -- 172.21.15.105:0/2055792935 wait complete. 2024-09-21T01:45:00.380 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705668 got 38654705667 for osd.1 2024-09-21T01:45:00.384 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705668 got 38654705667 for osd.3 2024-09-21T01:45:00.433 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.433+0000 7fa491176640 1 -- 172.21.15.105:0/275763910 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) v1 -- 0x7fa48c06aeb0 con 0x7fa48c128df0 2024-09-21T01:45:00.434 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.433+0000 7fa4737fe640 1 -- 172.21.15.105:0/275763910 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7fa48003e580 con 0x7fa48c128df0 2024-09-21T01:45:00.434 INFO:teuthology.orchestra.run.smithi105.stdout:38654705668 2024-09-21T01:45:00.435 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.435+0000 7fa491176640 1 -- 172.21.15.105:0/275763910 >> v1:172.21.15.105:6816/3933313069 conn(0x7fa478047560 legacy=0x7fa478049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:00.436 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.435+0000 7fa491176640 1 -- 172.21.15.105:0/275763910 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa48c128df0 msgr2=0x7fa48c1300a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:00.436 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.435+0000 7fa491176640 1 --2- 172.21.15.105:0/275763910 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa48c128df0 0x7fa48c1300a0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7fa48000e9f0 tx=0x7fa48000eeb0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.436 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.435+0000 7fa491176640 1 -- 172.21.15.105:0/275763910 shutdown_connections 2024-09-21T01:45:00.436 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.435+0000 7fa491176640 1 --2- 172.21.15.105:0/275763910 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa48c12c430 0x7fa48c1305e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.436 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.435+0000 7fa491176640 1 --2- 172.21.15.105:0/275763910 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa48c128df0 0x7fa48c1300a0 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.436 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.435+0000 7fa491176640 1 --2- 172.21.15.105:0/275763910 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa48c09daa0 0x7fa48c12fb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:00.436 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.435+0000 7fa491176640 1 -- 172.21.15.105:0/275763910 >> 172.21.15.105:0/275763910 conn(0x7fa48c09b7c0 msgr2=0x7fa48c099310 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:00.436 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.435+0000 7fa491176640 1 -- 172.21.15.105:0/275763910 shutdown_connections 2024-09-21T01:45:00.436 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:00.435+0000 7fa491176640 1 -- 172.21.15.105:0/275763910 wait complete. 2024-09-21T01:45:00.446 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705669 got 38654705668 for osd.0 2024-09-21T01:45:01.341 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T01:45:01.381 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T01:45:01.384 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T01:45:01.446 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T01:45:01.458 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.458+0000 7f866243d640 1 -- 172.21.15.105:0/2548422585 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f865c126090 msgr2=0x7f865c128c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.458 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.458+0000 7f866243d640 1 --2- 172.21.15.105:0/2548422585 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f865c126090 0x7f865c128c70 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f8650005fd0 tx=0x7f865002a410 comp rx=0 tx=0).stop 2024-09-21T01:45:01.458 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.458+0000 7f866243d640 1 -- 172.21.15.105:0/2548422585 shutdown_connections 2024-09-21T01:45:01.458 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.458+0000 7f866243d640 1 --2- 172.21.15.105:0/2548422585 >> v2:172.21.15.105:3300/0 conn(0x7f865c1291b0 0x7f865c129610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.458 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.458+0000 7f866243d640 1 --2- 172.21.15.105:0/2548422585 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f865c126090 0x7f865c128c70 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.458 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.458+0000 7f866243d640 1 --2- 172.21.15.105:0/2548422585 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f865c125770 0x7f865c125b50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.459 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.458+0000 7f866243d640 1 -- 172.21.15.105:0/2548422585 >> 172.21.15.105:0/2548422585 conn(0x7f865c099480 msgr2=0x7f865c099890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:01.459 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.458+0000 7f866243d640 1 -- 172.21.15.105:0/2548422585 shutdown_connections 2024-09-21T01:45:01.459 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.458+0000 7f866243d640 1 -- 172.21.15.105:0/2548422585 wait complete. 2024-09-21T01:45:01.459 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.459+0000 7f866243d640 1 Processor -- start 2024-09-21T01:45:01.460 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.459+0000 7f866243d640 1 -- start start 2024-09-21T01:45:01.460 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.459+0000 7f866243d640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f865c125770 0x7f865c089000 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:01.460 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.459+0000 7f866243d640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f865c126090 0x7f865c089540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:01.460 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.459+0000 7f866243d640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f865c1291b0 0x7f865c089a80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:01.460 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.459+0000 7f866243d640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f865c08a3a0 con 0x7f865c125770 2024-09-21T01:45:01.461 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.459+0000 7f866243d640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f865c08b6e0 con 0x7f865c126090 2024-09-21T01:45:01.461 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.459+0000 7f866243d640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f865c08ea10 con 0x7f865c1291b0 2024-09-21T01:45:01.461 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.459+0000 7f86609da640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f865c1291b0 0x7f865c089a80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:01.461 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.459+0000 7f865bfff640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f865c125770 0x7f865c089000 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:01.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.460+0000 7f86609da640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f865c1291b0 0x7f865c089a80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:38574/0 (socket says 172.21.15.105:38574) 2024-09-21T01:45:01.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.460+0000 7f865b7fe640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f865c126090 0x7f865c089540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:01.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.460+0000 7f86609da640 1 -- 172.21.15.105:0/726578106 learned_addr learned my addr 172.21.15.105:0/726578106 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:45:01.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.460+0000 7f865bfff640 1 -- 172.21.15.105:0/726578106 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f865c1291b0 msgr2=0x7f865c089a80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.460+0000 7f865bfff640 1 --2- 172.21.15.105:0/726578106 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f865c1291b0 0x7f865c089a80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.460+0000 7f865bfff640 1 -- 172.21.15.105:0/726578106 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f865c126090 msgr2=0x7f865c089540 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.460+0000 7f865bfff640 1 --2- 172.21.15.105:0/726578106 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f865c126090 0x7f865c089540 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.460+0000 7f865bfff640 1 -- 172.21.15.105:0/726578106 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8654009840 con 0x7f865c125770 2024-09-21T01:45:01.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.460+0000 7f86609da640 1 --2- 172.21.15.105:0/726578106 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f865c1291b0 0x7f865c089a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:45:01.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.460+0000 7f865b7fe640 1 --2- 172.21.15.105:0/726578106 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f865c126090 0x7f865c089540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:45:01.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.460+0000 7f865bfff640 1 --2- 172.21.15.105:0/726578106 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f865c125770 0x7f865c089000 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f864c00bfd0 tx=0x7f864c00da70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:45:01.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.460+0000 7f8658ff9640 1 -- 172.21.15.105:0/726578106 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f864c0098f0 con 0x7f865c125770 2024-09-21T01:45:01.463 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.460+0000 7f8658ff9640 1 -- 172.21.15.105:0/726578106 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f864c009b40 con 0x7f865c125770 2024-09-21T01:45:01.463 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.461+0000 7f866243d640 1 -- 172.21.15.105:0/726578106 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8650009560 con 0x7f865c125770 2024-09-21T01:45:01.463 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.461+0000 7f8658ff9640 1 -- 172.21.15.105:0/726578106 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f864c010460 con 0x7f865c125770 2024-09-21T01:45:01.463 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.461+0000 7f866243d640 1 -- 172.21.15.105:0/726578106 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f865c125460 con 0x7f865c125770 2024-09-21T01:45:01.463 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.462+0000 7f8658ff9640 1 -- 172.21.15.105:0/726578106 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f864c0105c0 con 0x7f865c125770 2024-09-21T01:45:01.464 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.462+0000 7f8658ff9640 1 -- 172.21.15.105:0/726578106 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f864c061d10 con 0x7f865c125770 2024-09-21T01:45:01.464 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.463+0000 7f866243d640 1 -- 172.21.15.105:0/726578106 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f865c125e50 con 0x7f865c125770 2024-09-21T01:45:01.470 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.470+0000 7f8658ff9640 1 -- 172.21.15.105:0/726578106 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f864c017070 con 0x7f865c125770 2024-09-21T01:45:01.497 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.497+0000 7f42fc7ec640 1 -- 172.21.15.105:0/3989495594 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f42f412c430 msgr2=0x7f42f412c810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.498 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.497+0000 7f42fc7ec640 1 --2- 172.21.15.105:0/3989495594 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f42f412c430 0x7f42f412c810 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7f42f0005fd0 tx=0x7f42f002a4b0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.498 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.497+0000 7f42fc7ec640 1 -- 172.21.15.105:0/3989495594 shutdown_connections 2024-09-21T01:45:01.498 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.497+0000 7f42fc7ec640 1 --2- 172.21.15.105:0/3989495594 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f42f409daa0 0x7f42f41288b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.498 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.497+0000 7f42fc7ec640 1 --2- 172.21.15.105:0/3989495594 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f42f412c430 0x7f42f412c810 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.498 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.497+0000 7f42fc7ec640 1 -- 172.21.15.105:0/3989495594 >> 172.21.15.105:0/3989495594 conn(0x7f42f409b7c0 msgr2=0x7f42f409bbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:01.498 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.497+0000 7f42fc7ec640 1 -- 172.21.15.105:0/3989495594 shutdown_connections 2024-09-21T01:45:01.498 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.497+0000 7f42fc7ec640 1 -- 172.21.15.105:0/3989495594 wait complete. 2024-09-21T01:45:01.502 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.498+0000 7f42fc7ec640 1 Processor -- start 2024-09-21T01:45:01.502 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.498+0000 7f42fc7ec640 1 -- start start 2024-09-21T01:45:01.502 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.498+0000 7f42fc7ec640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f42f409daa0 0x7f42f412fd00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:01.502 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.498+0000 7f42fc7ec640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f42f4128df0 0x7f42f4130240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:01.503 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.498+0000 7f42fc7ec640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f42f412c430 0x7f42f4130780 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:01.503 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.498+0000 7f42fc7ec640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f42f4132fb0 con 0x7f42f409daa0 2024-09-21T01:45:01.503 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.498+0000 7f42fc7ec640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f42f41327d0 con 0x7f42f4128df0 2024-09-21T01:45:01.503 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.498+0000 7f42fc7ec640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f42f4134310 con 0x7f42f412c430 2024-09-21T01:45:01.503 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.498+0000 7f42fad89640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f42f412c430 0x7f42f4130780 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:01.503 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.498+0000 7f42fad89640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f42f412c430 0x7f42f4130780 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:38596/0 (socket says 172.21.15.105:38596) 2024-09-21T01:45:01.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.498+0000 7f42fad89640 1 -- 172.21.15.105:0/3458200979 learned_addr learned my addr 172.21.15.105:0/3458200979 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:45:01.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.498+0000 7f42fa588640 1 --2- 172.21.15.105:0/3458200979 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f42f409daa0 0x7f42f412fd00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:01.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.498+0000 7f42f9d87640 1 --2- 172.21.15.105:0/3458200979 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f42f4128df0 0x7f42f4130240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:01.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.499+0000 7f42fad89640 1 -- 172.21.15.105:0/3458200979 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f42f4128df0 msgr2=0x7f42f4130240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.499+0000 7f42fad89640 1 --2- 172.21.15.105:0/3458200979 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f42f4128df0 0x7f42f4130240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.499+0000 7f42fad89640 1 -- 172.21.15.105:0/3458200979 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f42f409daa0 msgr2=0x7f42f412fd00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.499+0000 7f42fad89640 1 --2- 172.21.15.105:0/3458200979 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f42f409daa0 0x7f42f412fd00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.499+0000 7f42fad89640 1 -- 172.21.15.105:0/3458200979 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f42f0009560 con 0x7f42f412c430 2024-09-21T01:45:01.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.499+0000 7f42f9d87640 1 --2- 172.21.15.105:0/3458200979 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f42f4128df0 0x7f42f4130240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:45:01.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.499+0000 7f42fa588640 1 --2- 172.21.15.105:0/3458200979 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f42f409daa0 0x7f42f412fd00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:45:01.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.499+0000 7f42fad89640 1 --2- 172.21.15.105:0/3458200979 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f42f412c430 0x7f42f4130780 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f42ec00c780 tx=0x7f42ec00cc40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:45:01.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.499+0000 7f42eaffd640 1 -- 172.21.15.105:0/3458200979 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f42ec003da0 con 0x7f42f412c430 2024-09-21T01:45:01.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.499+0000 7f42fc7ec640 1 -- 172.21.15.105:0/3458200979 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f42f4131f90 con 0x7f42f412c430 2024-09-21T01:45:01.505 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.499+0000 7f42eaffd640 1 -- 172.21.15.105:0/3458200979 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f42ec00ce90 con 0x7f42f412c430 2024-09-21T01:45:01.505 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.499+0000 7f42eaffd640 1 -- 172.21.15.105:0/3458200979 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f42ec0052b0 con 0x7f42f412c430 2024-09-21T01:45:01.505 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.499+0000 7f42fc7ec640 1 -- 172.21.15.105:0/3458200979 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f42f4130cc0 con 0x7f42f412c430 2024-09-21T01:45:01.505 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.501+0000 7f42fc7ec640 1 -- 172.21.15.105:0/3458200979 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f42f4130fa0 con 0x7f42f412c430 2024-09-21T01:45:01.505 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.501+0000 7f42eaffd640 1 -- 172.21.15.105:0/3458200979 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f42ec005410 con 0x7f42f412c430 2024-09-21T01:45:01.505 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.501+0000 7f42eaffd640 1 -- 172.21.15.105:0/3458200979 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f42ec05c060 con 0x7f42f412c430 2024-09-21T01:45:01.509 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.509+0000 7f42eaffd640 1 -- 172.21.15.105:0/3458200979 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f42ec039cf0 con 0x7f42f412c430 2024-09-21T01:45:01.509 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.509+0000 7f2feeffd640 1 -- 172.21.15.105:0/1440507917 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2fe8035bc0 con 0x7f2ff01271a0 2024-09-21T01:45:01.510 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.510+0000 7f2ff6725640 1 -- 172.21.15.105:0/1440507917 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2ff01271a0 msgr2=0x7f2ff0131ef0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.510 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.510+0000 7f2ff6725640 1 --2- 172.21.15.105:0/1440507917 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2ff01271a0 0x7f2ff0131ef0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f2fe8009c10 tx=0x7f2fe802e900 comp rx=0 tx=0).stop 2024-09-21T01:45:01.512 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.511+0000 7f2ff6725640 1 -- 172.21.15.105:0/1440507917 shutdown_connections 2024-09-21T01:45:01.512 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.511+0000 7f2ff6725640 1 --2- 172.21.15.105:0/1440507917 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2ff01271a0 0x7f2ff0131ef0 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.512 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.511+0000 7f2ff6725640 1 --2- 172.21.15.105:0/1440507917 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2ff012a030 0x7f2ff012a410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.512 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.511+0000 7f2ff6725640 1 -- 172.21.15.105:0/1440507917 >> 172.21.15.105:0/1440507917 conn(0x7f2ff009c6e0 msgr2=0x7f2ff009caf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:01.512 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.511+0000 7f2ff6725640 1 -- 172.21.15.105:0/1440507917 shutdown_connections 2024-09-21T01:45:01.512 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.511+0000 7f2ff6725640 1 -- 172.21.15.105:0/1440507917 wait complete. 2024-09-21T01:45:01.512 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.512+0000 7f2ff6725640 1 Processor -- start 2024-09-21T01:45:01.513 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.512+0000 7f2ff6725640 1 -- start start 2024-09-21T01:45:01.513 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.513+0000 7f2ff6725640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2ff0126650 0x7f2ff01175e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:01.513 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.513+0000 7f2ff6725640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2ff01271a0 0x7f2ff0117b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:01.513 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.513+0000 7f2ff6725640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2ff012a030 0x7f2ff0118060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:01.513 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.513+0000 7f2ff6725640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2ff0119b50 con 0x7f2ff012a030 2024-09-21T01:45:01.513 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.513+0000 7f2ff6725640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f2ff01243b0 con 0x7f2ff0126650 2024-09-21T01:45:01.513 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.513+0000 7f2ff6725640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f2ff011adb0 con 0x7f2ff01271a0 2024-09-21T01:45:01.514 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.513+0000 7f2fef7fe640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2ff01271a0 0x7f2ff0117b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:01.514 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.513+0000 7f2ff4cc2640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2ff012a030 0x7f2ff0118060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:01.514 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.513+0000 7f2feffff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2ff0126650 0x7f2ff01175e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:01.514 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.513+0000 7f2ff4cc2640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2ff012a030 0x7f2ff0118060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:40372/0 (socket says 172.21.15.105:40372) 2024-09-21T01:45:01.514 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.513+0000 7f2ff4cc2640 1 -- 172.21.15.105:0/511252595 learned_addr learned my addr 172.21.15.105:0/511252595 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:45:01.515 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.513+0000 7f2ff4cc2640 1 -- 172.21.15.105:0/511252595 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2ff01271a0 msgr2=0x7f2ff0117b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.515 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.513+0000 7f2ff4cc2640 1 --2- 172.21.15.105:0/511252595 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2ff01271a0 0x7f2ff0117b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.515 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.513+0000 7f2ff4cc2640 1 -- 172.21.15.105:0/511252595 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2ff0126650 msgr2=0x7f2ff01175e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.515 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.513+0000 7f2ff4cc2640 1 --2- 172.21.15.105:0/511252595 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2ff0126650 0x7f2ff01175e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.515 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.513+0000 7f2ff4cc2640 1 -- 172.21.15.105:0/511252595 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2fe0009560 con 0x7f2ff012a030 2024-09-21T01:45:01.515 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.513+0000 7f2feffff640 1 --2- 172.21.15.105:0/511252595 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2ff0126650 0x7f2ff01175e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:45:01.516 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.514+0000 7f2ff4cc2640 1 --2- 172.21.15.105:0/511252595 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2ff012a030 0x7f2ff0118060 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f2fe802eef0 tx=0x7f2fe8036b60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:45:01.516 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.514+0000 7f2fecff9640 1 -- 172.21.15.105:0/511252595 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2fe8004240 con 0x7f2ff012a030 2024-09-21T01:45:01.516 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.514+0000 7f2fecff9640 1 -- 172.21.15.105:0/511252595 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f2fe8004490 con 0x7f2ff012a030 2024-09-21T01:45:01.516 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.514+0000 7f2fecff9640 1 -- 172.21.15.105:0/511252595 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f2fe8049460 con 0x7f2ff012a030 2024-09-21T01:45:01.516 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.514+0000 7f2ff6725640 1 -- 172.21.15.105:0/511252595 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2fe8009840 con 0x7f2ff012a030 2024-09-21T01:45:01.516 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.515+0000 7f2ff6725640 1 -- 172.21.15.105:0/511252595 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2ff01185a0 con 0x7f2ff012a030 2024-09-21T01:45:01.516 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.515+0000 7f2fecff9640 1 -- 172.21.15.105:0/511252595 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f2fe8040420 con 0x7f2ff012a030 2024-09-21T01:45:01.516 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.516+0000 7f2fecff9640 1 -- 172.21.15.105:0/511252595 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f2fe8085d80 con 0x7f2ff012a030 2024-09-21T01:45:01.516 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.516+0000 7f2ff6725640 1 -- 172.21.15.105:0/511252595 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2ff012a410 con 0x7f2ff012a030 2024-09-21T01:45:01.520 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.520+0000 7f2fecff9640 1 -- 172.21.15.105:0/511252595 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f2fe805f9c0 con 0x7f2ff012a030 2024-09-21T01:45:01.559 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.558+0000 7f16abf08640 1 -- 172.21.15.105:0/3431713293 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f16a409daa0 msgr2=0x7f16a41288b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.559 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.558+0000 7f16abf08640 1 --2- 172.21.15.105:0/3431713293 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f16a409daa0 0x7f16a41288b0 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f1694005fd0 tx=0x7f169402a420 comp rx=0 tx=0).stop 2024-09-21T01:45:01.559 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.558+0000 7f16abf08640 1 -- 172.21.15.105:0/3431713293 shutdown_connections 2024-09-21T01:45:01.559 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.558+0000 7f16abf08640 1 --2- 172.21.15.105:0/3431713293 >> v2:172.21.15.105:3300/0 conn(0x7f16a4128df0 0x7f16a41292a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.559 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.558+0000 7f16abf08640 1 --2- 172.21.15.105:0/3431713293 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f16a409daa0 0x7f16a41288b0 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.559 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.558+0000 7f16abf08640 1 --2- 172.21.15.105:0/3431713293 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f16a412c430 0x7f16a412c810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.560 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.558+0000 7f16abf08640 1 -- 172.21.15.105:0/3431713293 >> 172.21.15.105:0/3431713293 conn(0x7f16a409b7c0 msgr2=0x7f16a409bbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:01.560 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.559+0000 7f16abf08640 1 -- 172.21.15.105:0/3431713293 shutdown_connections 2024-09-21T01:45:01.560 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.559+0000 7f16abf08640 1 -- 172.21.15.105:0/3431713293 wait complete. 2024-09-21T01:45:01.563 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.559+0000 7f16abf08640 1 Processor -- start 2024-09-21T01:45:01.563 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.560+0000 7f16abf08640 1 -- start start 2024-09-21T01:45:01.563 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.560+0000 7f16abf08640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f16a409daa0 0x7f16a4121890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:01.564 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.560+0000 7f16abf08640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f16a4128df0 0x7f16a4121dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:01.564 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.560+0000 7f16abf08640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f16a412c430 0x7f16a412fcf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:01.564 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.560+0000 7f16abf08640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f16a4131770 con 0x7f16a4128df0 2024-09-21T01:45:01.564 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.560+0000 7f16a9ca4640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f16a409daa0 0x7f16a4121890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:01.564 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.560+0000 7f16abf08640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f16a4126580 con 0x7f16a409daa0 2024-09-21T01:45:01.565 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.560+0000 7f16a9ca4640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f16a409daa0 0x7f16a4121890 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:59832/0 (socket says 172.21.15.105:59832) 2024-09-21T01:45:01.565 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.560+0000 7f16a9ca4640 1 -- 172.21.15.105:0/796672630 learned_addr learned my addr 172.21.15.105:0/796672630 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:45:01.565 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.560+0000 7f16a94a3640 1 --2- 172.21.15.105:0/796672630 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f16a4128df0 0x7f16a4121dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:01.565 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.560+0000 7f16abf08640 1 -- 172.21.15.105:0/796672630 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f16a4132b00 con 0x7f16a412c430 2024-09-21T01:45:01.565 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.560+0000 7f16aa4a5640 1 --2- 172.21.15.105:0/796672630 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f16a412c430 0x7f16a412fcf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:01.565 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.561+0000 7f16a9ca4640 1 -- 172.21.15.105:0/796672630 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f16a412c430 msgr2=0x7f16a412fcf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.565 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.561+0000 7f16a9ca4640 1 --2- 172.21.15.105:0/796672630 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f16a412c430 0x7f16a412fcf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.565 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.561+0000 7f16a9ca4640 1 -- 172.21.15.105:0/796672630 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f16a4128df0 msgr2=0x7f16a4121dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.565 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.561+0000 7f16a9ca4640 1 --2- 172.21.15.105:0/796672630 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f16a4128df0 0x7f16a4121dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.565 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.561+0000 7f16a9ca4640 1 -- 172.21.15.105:0/796672630 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1694009560 con 0x7f16a409daa0 2024-09-21T01:45:01.565 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.561+0000 7f16a9ca4640 1 --2- 172.21.15.105:0/796672630 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f16a409daa0 0x7f16a4121890 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f16a000d950 tx=0x7f16a000de10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:45:01.565 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.561+0000 7f169a7fc640 1 -- 172.21.15.105:0/796672630 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16a0009c30 con 0x7f16a409daa0 2024-09-21T01:45:01.565 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.561+0000 7f169a7fc640 1 -- 172.21.15.105:0/796672630 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f16a0009e80 con 0x7f16a409daa0 2024-09-21T01:45:01.565 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.561+0000 7f169a7fc640 1 -- 172.21.15.105:0/796672630 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f16a00106c0 con 0x7f16a409daa0 2024-09-21T01:45:01.566 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.561+0000 7f16abf08640 1 -- 172.21.15.105:0/796672630 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f16a4122310 con 0x7f16a409daa0 2024-09-21T01:45:01.566 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.562+0000 7f16abf08640 1 -- 172.21.15.105:0/796672630 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f16a4130230 con 0x7f16a409daa0 2024-09-21T01:45:01.566 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.563+0000 7f169a7fc640 1 -- 172.21.15.105:0/796672630 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f16a0010820 con 0x7f16a409daa0 2024-09-21T01:45:01.566 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.563+0000 7f16abf08640 1 -- 172.21.15.105:0/796672630 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f16a41224a0 con 0x7f16a409daa0 2024-09-21T01:45:01.566 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.564+0000 7f169a7fc640 1 -- 172.21.15.105:0/796672630 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f16a0061d10 con 0x7f16a409daa0 2024-09-21T01:45:01.567 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.567+0000 7f169a7fc640 1 -- 172.21.15.105:0/796672630 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f16a0062060 con 0x7f16a409daa0 2024-09-21T01:45:01.636 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.636+0000 7f866243d640 1 -- 172.21.15.105:0/726578106 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) v1 -- 0x7f865c06ad10 con 0x7f865c125770 2024-09-21T01:45:01.637 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.636+0000 7f8658ff9640 1 -- 172.21.15.105:0/726578106 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f864c062060 con 0x7f865c125770 2024-09-21T01:45:01.637 INFO:teuthology.orchestra.run.smithi105.stdout:38654705669 2024-09-21T01:45:01.640 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.639+0000 7f866243d640 1 -- 172.21.15.105:0/726578106 >> v1:172.21.15.105:6816/3933313069 conn(0x7f8648047560 legacy=0x7f8648049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.640 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.639+0000 7f866243d640 1 -- 172.21.15.105:0/726578106 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f865c125770 msgr2=0x7f865c089000 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.640 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.639+0000 7f866243d640 1 --2- 172.21.15.105:0/726578106 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f865c125770 0x7f865c089000 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f864c00bfd0 tx=0x7f864c00da70 comp rx=0 tx=0).stop 2024-09-21T01:45:01.640 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.639+0000 7f866243d640 1 -- 172.21.15.105:0/726578106 shutdown_connections 2024-09-21T01:45:01.640 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.639+0000 7f866243d640 1 --2- 172.21.15.105:0/726578106 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f865c1291b0 0x7f865c089a80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.640 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.639+0000 7f866243d640 1 --2- 172.21.15.105:0/726578106 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f865c126090 0x7f865c089540 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.640 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.639+0000 7f866243d640 1 --2- 172.21.15.105:0/726578106 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f865c125770 0x7f865c089000 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.640 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.639+0000 7f866243d640 1 -- 172.21.15.105:0/726578106 >> 172.21.15.105:0/726578106 conn(0x7f865c099480 msgr2=0x7f865c095f80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:01.640 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.639+0000 7f866243d640 1 -- 172.21.15.105:0/726578106 shutdown_connections 2024-09-21T01:45:01.640 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.639+0000 7f866243d640 1 -- 172.21.15.105:0/726578106 wait complete. 2024-09-21T01:45:01.654 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705668 got 38654705669 for osd.2 2024-09-21T01:45:01.654 DEBUG:teuthology.parallel:result is None 2024-09-21T01:45:01.656 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.656+0000 7f42fc7ec640 1 -- 172.21.15.105:0/3458200979 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) v1 -- 0x7f42f4098e40 con 0x7f42f412c430 2024-09-21T01:45:01.656 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.656+0000 7f42eaffd640 1 -- 172.21.15.105:0/3458200979 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f42f4098e40 con 0x7f42f412c430 2024-09-21T01:45:01.656 INFO:teuthology.orchestra.run.smithi105.stdout:38654705669 2024-09-21T01:45:01.658 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.658+0000 7f42fc7ec640 1 -- 172.21.15.105:0/3458200979 >> v1:172.21.15.105:6816/3933313069 conn(0x7f42d8047590 legacy=0x7f42d8049a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.658 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.658+0000 7f42fc7ec640 1 -- 172.21.15.105:0/3458200979 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f42f412c430 msgr2=0x7f42f4130780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.658 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.658+0000 7f42fc7ec640 1 --2- 172.21.15.105:0/3458200979 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f42f412c430 0x7f42f4130780 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f42ec00c780 tx=0x7f42ec00cc40 comp rx=0 tx=0).stop 2024-09-21T01:45:01.658 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.658+0000 7f42fc7ec640 1 -- 172.21.15.105:0/3458200979 shutdown_connections 2024-09-21T01:45:01.658 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.658+0000 7f42fc7ec640 1 --2- 172.21.15.105:0/3458200979 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f42f412c430 0x7f42f4130780 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.659 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.658+0000 7f42fc7ec640 1 --2- 172.21.15.105:0/3458200979 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f42f4128df0 0x7f42f4130240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.659 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.658+0000 7f42fc7ec640 1 --2- 172.21.15.105:0/3458200979 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f42f409daa0 0x7f42f412fd00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.659 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.658+0000 7f42fc7ec640 1 -- 172.21.15.105:0/3458200979 >> 172.21.15.105:0/3458200979 conn(0x7f42f409b7c0 msgr2=0x7f42f4127e60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:01.659 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.658+0000 7f42fc7ec640 1 -- 172.21.15.105:0/3458200979 shutdown_connections 2024-09-21T01:45:01.659 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.658+0000 7f42fc7ec640 1 -- 172.21.15.105:0/3458200979 wait complete. 2024-09-21T01:45:01.664 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.663+0000 7f2ff6725640 1 -- 172.21.15.105:0/511252595 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) v1 -- 0x7f2ff0118810 con 0x7f2ff012a030 2024-09-21T01:45:01.664 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.664+0000 7f2fecff9640 1 -- 172.21.15.105:0/511252595 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f2fe804e020 con 0x7f2ff012a030 2024-09-21T01:45:01.664 INFO:teuthology.orchestra.run.smithi105.stdout:38654705669 2024-09-21T01:45:01.666 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.665+0000 7f2ff6725640 1 -- 172.21.15.105:0/511252595 >> v1:172.21.15.105:6816/3933313069 conn(0x7f2fdc047560 legacy=0x7f2fdc049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.666 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.665+0000 7f2ff6725640 1 -- 172.21.15.105:0/511252595 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2ff012a030 msgr2=0x7f2ff0118060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.666 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.665+0000 7f2ff6725640 1 --2- 172.21.15.105:0/511252595 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2ff012a030 0x7f2ff0118060 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f2fe802eef0 tx=0x7f2fe8036b60 comp rx=0 tx=0).stop 2024-09-21T01:45:01.666 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.665+0000 7f2ff6725640 1 -- 172.21.15.105:0/511252595 shutdown_connections 2024-09-21T01:45:01.666 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.665+0000 7f2ff6725640 1 --2- 172.21.15.105:0/511252595 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2ff012a030 0x7f2ff0118060 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.666 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.665+0000 7f2ff6725640 1 --2- 172.21.15.105:0/511252595 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2ff01271a0 0x7f2ff0117b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.666 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.665+0000 7f2ff6725640 1 --2- 172.21.15.105:0/511252595 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2ff0126650 0x7f2ff01175e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.667 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.665+0000 7f2ff6725640 1 -- 172.21.15.105:0/511252595 >> 172.21.15.105:0/511252595 conn(0x7f2ff009c6e0 msgr2=0x7f2ff0128ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:01.667 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.665+0000 7f2ff6725640 1 -- 172.21.15.105:0/511252595 shutdown_connections 2024-09-21T01:45:01.667 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.665+0000 7f2ff6725640 1 -- 172.21.15.105:0/511252595 wait complete. 2024-09-21T01:45:01.669 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705668 got 38654705669 for osd.3 2024-09-21T01:45:01.669 DEBUG:teuthology.parallel:result is None 2024-09-21T01:45:01.676 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705668 got 38654705669 for osd.1 2024-09-21T01:45:01.676 DEBUG:teuthology.parallel:result is None 2024-09-21T01:45:01.717 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.716+0000 7f16abf08640 1 -- 172.21.15.105:0/796672630 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) v1 -- 0x7f16a412a2c0 con 0x7f16a409daa0 2024-09-21T01:45:01.717 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.716+0000 7f169a7fc640 1 -- 172.21.15.105:0/796672630 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f16a0017070 con 0x7f16a409daa0 2024-09-21T01:45:01.717 INFO:teuthology.orchestra.run.smithi105.stdout:38654705670 2024-09-21T01:45:01.719 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.718+0000 7f16abf08640 1 -- 172.21.15.105:0/796672630 >> v1:172.21.15.105:6816/3933313069 conn(0x7f1688047560 legacy=0x7f1688049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.719 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.718+0000 7f16abf08640 1 -- 172.21.15.105:0/796672630 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f16a409daa0 msgr2=0x7f16a4121890 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.719 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.718+0000 7f16abf08640 1 --2- 172.21.15.105:0/796672630 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f16a409daa0 0x7f16a4121890 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f16a000d950 tx=0x7f16a000de10 comp rx=0 tx=0).stop 2024-09-21T01:45:01.719 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.718+0000 7f16abf08640 1 -- 172.21.15.105:0/796672630 shutdown_connections 2024-09-21T01:45:01.719 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.718+0000 7f16abf08640 1 --2- 172.21.15.105:0/796672630 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f16a412c430 0x7f16a412fcf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.719 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.718+0000 7f16abf08640 1 --2- 172.21.15.105:0/796672630 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f16a4128df0 0x7f16a4121dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.719 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.718+0000 7f16abf08640 1 --2- 172.21.15.105:0/796672630 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f16a409daa0 0x7f16a4121890 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.720 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.718+0000 7f16abf08640 1 -- 172.21.15.105:0/796672630 >> 172.21.15.105:0/796672630 conn(0x7f16a409b7c0 msgr2=0x7f16a412a800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:01.720 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.718+0000 7f16abf08640 1 -- 172.21.15.105:0/796672630 shutdown_connections 2024-09-21T01:45:01.720 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.718+0000 7f16abf08640 1 -- 172.21.15.105:0/796672630 wait complete. 2024-09-21T01:45:01.729 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705669 got 38654705670 for osd.0 2024-09-21T01:45:01.729 DEBUG:teuthology.parallel:result is None 2024-09-21T01:45:01.729 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean 2024-09-21T01:45:01.729 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T01:45:01.871 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.868+0000 7f49ac632640 1 -- 172.21.15.105:0/197106315 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f49a40928a0 msgr2=0x7f49a4092c80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.871 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.868+0000 7f49ac632640 1 --2- 172.21.15.105:0/197106315 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f49a40928a0 0x7f49a4092c80 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f49a0009830 tx=0x7f49a002a480 comp rx=0 tx=0).stop 2024-09-21T01:45:01.871 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.868+0000 7f49ac632640 1 -- 172.21.15.105:0/197106315 shutdown_connections 2024-09-21T01:45:01.871 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.868+0000 7f49ac632640 1 --2- 172.21.15.105:0/197106315 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f49a4128d10 0x7f49a4129140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.871 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.868+0000 7f49ac632640 1 --2- 172.21.15.105:0/197106315 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f49a40928a0 0x7f49a4092c80 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.871 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.868+0000 7f49ac632640 1 -- 172.21.15.105:0/197106315 >> 172.21.15.105:0/197106315 conn(0x7f49a4098760 msgr2=0x7f49a4098b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:01.872 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.868+0000 7f49ac632640 1 -- 172.21.15.105:0/197106315 shutdown_connections 2024-09-21T01:45:01.872 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.868+0000 7f49ac632640 1 -- 172.21.15.105:0/197106315 wait complete. 2024-09-21T01:45:01.872 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.869+0000 7f49ac632640 1 Processor -- start 2024-09-21T01:45:01.872 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.869+0000 7f49ac632640 1 -- start start 2024-09-21T01:45:01.872 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.869+0000 7f49ac632640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f49a40928a0 0x7f49a4070d40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:01.872 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.869+0000 7f49ac632640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f49a40931c0 0x7f49a4071280 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:01.872 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.869+0000 7f49ac632640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f49a4128d10 0x7f49a40717c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:01.872 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.869+0000 7f49ac632640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f49a4089d40 con 0x7f49a40931c0 2024-09-21T01:45:01.872 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.869+0000 7f49ac632640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f49a408a740 con 0x7f49a4128d10 2024-09-21T01:45:01.872 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.869+0000 7f49ac632640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f49a4090ed0 con 0x7f49a40928a0 2024-09-21T01:45:01.872 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.869+0000 7f49a9bcd640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f49a40931c0 0x7f49a4071280 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:01.872 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.869+0000 7f49aabcf640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f49a4128d10 0x7f49a40717c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:01.872 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.869+0000 7f49aa3ce640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f49a40928a0 0x7f49a4070d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:01.873 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.870+0000 7f49a9bcd640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f49a40931c0 0x7f49a4071280 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:40414/0 (socket says 172.21.15.105:40414) 2024-09-21T01:45:01.873 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.870+0000 7f49aabcf640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f49a4128d10 0x7f49a40717c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:59854/0 (socket says 172.21.15.105:59854) 2024-09-21T01:45:01.873 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.870+0000 7f49a9bcd640 1 -- 172.21.15.105:0/3928566910 learned_addr learned my addr 172.21.15.105:0/3928566910 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:45:01.873 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.870+0000 7f49aa3ce640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f49a40928a0 0x7f49a4070d40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:38640/0 (socket says 172.21.15.105:38640) 2024-09-21T01:45:01.873 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.870+0000 7f49a9bcd640 1 -- 172.21.15.105:0/3928566910 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f49a40928a0 msgr2=0x7f49a4070d40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.873 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.870+0000 7f49a9bcd640 1 --2- 172.21.15.105:0/3928566910 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f49a40928a0 0x7f49a4070d40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.873 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.870+0000 7f49a9bcd640 1 -- 172.21.15.105:0/3928566910 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f49a4128d10 msgr2=0x7f49a40717c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:01.873 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.870+0000 7f49a9bcd640 1 --2- 172.21.15.105:0/3928566910 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f49a4128d10 0x7f49a40717c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:01.873 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.870+0000 7f49a9bcd640 1 -- 172.21.15.105:0/3928566910 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f499c009840 con 0x7f49a40931c0 2024-09-21T01:45:01.873 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.870+0000 7f49a9bcd640 1 --2- 172.21.15.105:0/3928566910 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f49a40931c0 0x7f49a4071280 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f499400c530 tx=0x7f499400c930 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:45:01.873 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.870+0000 7f499affd640 1 -- 172.21.15.105:0/3928566910 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4994011790 con 0x7f49a40931c0 2024-09-21T01:45:01.873 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.870+0000 7f499affd640 1 -- 172.21.15.105:0/3928566910 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f49940119e0 con 0x7f49a40931c0 2024-09-21T01:45:01.874 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.870+0000 7f49ac632640 1 -- 172.21.15.105:0/3928566910 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f49a0009490 con 0x7f49a40931c0 2024-09-21T01:45:01.874 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.870+0000 7f499affd640 1 -- 172.21.15.105:0/3928566910 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f4994011db0 con 0x7f49a40931c0 2024-09-21T01:45:01.874 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.870+0000 7f49ac632640 1 -- 172.21.15.105:0/3928566910 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f49a412ffd0 con 0x7f49a40931c0 2024-09-21T01:45:01.874 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.871+0000 7f49ac632640 1 -- 172.21.15.105:0/3928566910 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f49a41302b0 con 0x7f49a40931c0 2024-09-21T01:45:01.875 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.871+0000 7f499affd640 1 -- 172.21.15.105:0/3928566910 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f499401c460 con 0x7f49a40931c0 2024-09-21T01:45:01.875 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.872+0000 7f499affd640 1 -- 172.21.15.105:0/3928566910 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(17..17 src has 1..17) v4 ==== 2868+0+0 (secure 0 0 0) 0x7f499405c6b0 con 0x7f49a40931c0 2024-09-21T01:45:01.875 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:01.875+0000 7f499affd640 1 -- 172.21.15.105:0/3928566910 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f499403a360 con 0x7f49a40931c0 2024-09-21T01:45:02.006 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.005+0000 7f49ac632640 1 -- 172.21.15.105:0/3928566910 --> v1:172.21.15.105:6816/3933313069 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f49a4092c80 con 0x7f4988047560 2024-09-21T01:45:02.007 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.007+0000 7f499affd640 1 -- 172.21.15.105:0/3928566910 <== mgr.4112 v1:172.21.15.105:6816/3933313069 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+34967 (unknown 2965378022 0 97175841) 0x7f49a4092c80 con 0x7f4988047560 2024-09-21T01:45:02.007 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:45:02.008 INFO:teuthology.orchestra.run.smithi105.stderr:dumped all 2024-09-21T01:45:02.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.008+0000 7f49ac632640 1 -- 172.21.15.105:0/3928566910 >> v1:172.21.15.105:6816/3933313069 conn(0x7f4988047560 legacy=0x7f4988049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:02.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.008+0000 7f49ac632640 1 -- 172.21.15.105:0/3928566910 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f49a40931c0 msgr2=0x7f49a4071280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:02.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.008+0000 7f49ac632640 1 --2- 172.21.15.105:0/3928566910 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f49a40931c0 0x7f49a4071280 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f499400c530 tx=0x7f499400c930 comp rx=0 tx=0).stop 2024-09-21T01:45:02.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.008+0000 7f49ac632640 1 -- 172.21.15.105:0/3928566910 shutdown_connections 2024-09-21T01:45:02.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.008+0000 7f49ac632640 1 --2- 172.21.15.105:0/3928566910 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f49a4128d10 0x7f49a40717c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.008+0000 7f49ac632640 1 --2- 172.21.15.105:0/3928566910 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f49a40931c0 0x7f49a4071280 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.008+0000 7f49ac632640 1 --2- 172.21.15.105:0/3928566910 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f49a40928a0 0x7f49a4070d40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.008+0000 7f49ac632640 1 -- 172.21.15.105:0/3928566910 >> 172.21.15.105:0/3928566910 conn(0x7f49a4098760 msgr2=0x7f49a408d6f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:02.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.008+0000 7f49ac632640 1 -- 172.21.15.105:0/3928566910 shutdown_connections 2024-09-21T01:45:02.010 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.008+0000 7f49ac632640 1 -- 172.21.15.105:0/3928566910 wait complete. 2024-09-21T01:45:02.019 INFO:teuthology.orchestra.run.smithi105.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2024-09-21T01:45:00.351166+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":109840,"kb_used_data":1744,"kb_used_omap":6,"kb_used_meta":108025,"kb_avail":377377520,"statfs":{"total":386547056640,"available":386434580480,"internally_reserved":0,"allocated":1785856,"data_stored":1345730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6374,"internal_metadata":110618394},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.839169"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T01:44:57.534395+0000","last_change":"2024-09-21T01:44:57.534395+0000","last_active":"2024-09-21T01:44:57.534395+0000","last_peered":"2024-09-21T01:44:57.534395+0000","last_clean":"2024-09-21T01:44:57.534395+0000","last_became_active":"2024-09-21T01:44:55.526300+0000","last_became_peered":"2024-09-21T01:44:55.526300+0000","last_unstale":"2024-09-21T01:44:57.534395+0000","last_undegraded":"2024-09-21T01:44:57.534395+0000","last_fullsized":"2024-09-21T01:44:57.534395+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:38.742537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040868700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T01:44:57.531141+0000","last_change":"2024-09-21T01:44:57.531141+0000","last_active":"2024-09-21T01:44:57.531141+0000","last_peered":"2024-09-21T01:44:57.531141+0000","last_clean":"2024-09-21T01:44:57.531141+0000","last_became_active":"2024-09-21T01:44:55.527636+0000","last_became_peered":"2024-09-21T01:44:55.527636+0000","last_unstale":"2024-09-21T01:44:57.531141+0000","last_undegraded":"2024-09-21T01:44:57.531141+0000","last_fullsized":"2024-09-21T01:44:57.531141+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:57:35.618887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055542700000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T01:44:57.535152+0000","last_change":"2024-09-21T01:44:57.535152+0000","last_active":"2024-09-21T01:44:57.535152+0000","last_peered":"2024-09-21T01:44:57.535152+0000","last_clean":"2024-09-21T01:44:57.535152+0000","last_became_active":"2024-09-21T01:44:55.526163+0000","last_became_peered":"2024-09-21T01:44:55.526163+0000","last_unstale":"2024-09-21T01:44:57.535152+0000","last_undegraded":"2024-09-21T01:44:57.535152+0000","last_fullsized":"2024-09-21T01:44:57.535152+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:57:35.618887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00119484,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T01:44:57.531202+0000","last_change":"2024-09-21T01:44:57.531202+0000","last_active":"2024-09-21T01:44:57.531202+0000","last_peered":"2024-09-21T01:44:57.531202+0000","last_clean":"2024-09-21T01:44:57.531202+0000","last_became_active":"2024-09-21T01:44:55.526110+0000","last_became_peered":"2024-09-21T01:44:55.526110+0000","last_unstale":"2024-09-21T01:44:57.531202+0000","last_undegraded":"2024-09-21T01:44:57.531202+0000","last_fullsized":"2024-09-21T01:44:57.531202+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:38.742537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063215099999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"17'2","reported_seq":17,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T01:44:57.535975+0000","last_change":"2024-09-21T01:44:57.535093+0000","last_active":"2024-09-21T01:44:57.535975+0000","last_peered":"2024-09-21T01:44:57.535975+0000","last_clean":"2024-09-21T01:44:57.535975+0000","last_became_active":"2024-09-21T01:44:55.529008+0000","last_became_peered":"2024-09-21T01:44:55.529008+0000","last_unstale":"2024-09-21T01:44:57.535975+0000","last_undegraded":"2024-09-21T01:44:57.535975+0000","last_fullsized":"2024-09-21T01:44:57.535975+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:43.118995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010245149999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T01:44:57.531535+0000","last_change":"2024-09-21T01:44:57.531535+0000","last_active":"2024-09-21T01:44:57.531535+0000","last_peered":"2024-09-21T01:44:57.531535+0000","last_clean":"2024-09-21T01:44:57.531535+0000","last_became_active":"2024-09-21T01:44:55.526110+0000","last_became_peered":"2024-09-21T01:44:55.526110+0000","last_unstale":"2024-09-21T01:44:57.531535+0000","last_undegraded":"2024-09-21T01:44:57.531535+0000","last_fullsized":"2024-09-21T01:44:57.531535+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T05:54:08.666566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080666500000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T01:44:57.534522+0000","last_change":"2024-09-21T01:44:57.534522+0000","last_active":"2024-09-21T01:44:57.534522+0000","last_peered":"2024-09-21T01:44:57.534522+0000","last_clean":"2024-09-21T01:44:57.534522+0000","last_became_active":"2024-09-21T01:44:55.528489+0000","last_became_peered":"2024-09-21T01:44:55.528489+0000","last_unstale":"2024-09-21T01:44:57.534522+0000","last_undegraded":"2024-09-21T01:44:57.534522+0000","last_fullsized":"2024-09-21T01:44:57.534522+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T07:40:38.634948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058759600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"15'1","reported_seq":16,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T01:44:57.534320+0000","last_change":"2024-09-21T01:44:57.534320+0000","last_active":"2024-09-21T01:44:57.534320+0000","last_peered":"2024-09-21T01:44:57.534320+0000","last_clean":"2024-09-21T01:44:57.534320+0000","last_became_active":"2024-09-21T01:44:55.525577+0000","last_became_peered":"2024-09-21T01:44:55.525577+0000","last_unstale":"2024-09-21T01:44:57.534320+0000","last_undegraded":"2024-09-21T01:44:57.534320+0000","last_fullsized":"2024-09-21T01:44:57.534320+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T03:49:16.598027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036256899999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"12'97","reported_seq":141,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-21T01:44:56.532376+0000","last_change":"2024-09-21T01:44:51.556547+0000","last_active":"2024-09-21T01:44:56.532376+0000","last_peered":"2024-09-21T01:44:56.532376+0000","last_clean":"2024-09-21T01:44:56.532376+0000","last_became_active":"2024-09-21T01:44:51.555535+0000","last_became_peered":"2024-09-21T01:44:51.555535+0000","last_unstale":"2024-09-21T01:44:56.532376+0000","last_undegraded":"2024-09-21T01:44:56.532376+0000","last_fullsized":"2024-09-21T01:44:56.532376+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:50.489745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:50.489745+0000","last_clean_scrub_stamp":"2024-09-21T01:44:50.489745+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T03:15:49.853900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":9,"seq":38654705669,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27172,"kb_used_data":148,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344668,"statfs":{"total":96636764160,"available":96608940032,"internally_reserved":0,"allocated":151552,"data_stored":41258,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34300000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28299999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36099999999999999}]}]},{"osd":2,"up_from":9,"seq":38654705669,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27168,"kb_used_data":144,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344672,"statfs":{"total":96636764160,"available":96608944128,"internally_reserved":0,"allocated":147456,"data_stored":41239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.313}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41099999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35899999999999999}]}]},{"osd":1,"up_from":9,"seq":38654705669,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27752,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344088,"statfs":{"total":96636764160,"available":96608346112,"internally_reserved":0,"allocated":745472,"data_stored":631626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42399999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32500000000000001}]}]},{"osd":0,"up_from":9,"seq":38654705670,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27748,"kb_used_data":724,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344092,"statfs":{"total":96636764160,"available":96608350208,"internally_reserved":0,"allocated":741376,"data_stored":631607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33300000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T01:45:02.020 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T01:45:02.121 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.120+0000 7fb54a6c9640 1 -- 172.21.15.105:0/1203496669 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb54412ece0 msgr2=0x7fb5441310d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:02.121 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.120+0000 7fb54a6c9640 1 --2- 172.21.15.105:0/1203496669 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb54412ece0 0x7fb5441310d0 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7fb53c005fd0 tx=0x7fb53c02a780 comp rx=0 tx=0).stop 2024-09-21T01:45:02.121 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.120+0000 7fb54a6c9640 1 -- 172.21.15.105:0/1203496669 shutdown_connections 2024-09-21T01:45:02.121 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.120+0000 7fb54a6c9640 1 --2- 172.21.15.105:0/1203496669 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb54412ece0 0x7fb5441310d0 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.121 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.120+0000 7fb54a6c9640 1 --2- 172.21.15.105:0/1203496669 >> v2:172.21.15.105:3300/0 conn(0x7fb544093740 0x7fb544093b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.121 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.120+0000 7fb54a6c9640 1 -- 172.21.15.105:0/1203496669 >> 172.21.15.105:0/1203496669 conn(0x7fb544098760 msgr2=0x7fb544098b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:02.122 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.120+0000 7fb54a6c9640 1 -- 172.21.15.105:0/1203496669 shutdown_connections 2024-09-21T01:45:02.122 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.120+0000 7fb54a6c9640 1 -- 172.21.15.105:0/1203496669 wait complete. 2024-09-21T01:45:02.126 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.121+0000 7fb54a6c9640 1 Processor -- start 2024-09-21T01:45:02.127 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.121+0000 7fb54a6c9640 1 -- start start 2024-09-21T01:45:02.127 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.121+0000 7fb54a6c9640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb544093740 0x7fb544089270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:02.127 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.121+0000 7fb54a6c9640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb544094060 0x7fb5440897b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:02.127 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.121+0000 7fb54a6c9640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb54412ece0 0x7fb544089cf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:02.127 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.121+0000 7fb54a6c9640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb54408aa90 con 0x7fb544094060 2024-09-21T01:45:02.128 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.121+0000 7fb54a6c9640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7fb544070d00 con 0x7fb544093740 2024-09-21T01:45:02.128 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.121+0000 7fb54a6c9640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7fb5440703d0 con 0x7fb54412ece0 2024-09-21T01:45:02.128 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.121+0000 7fb543fff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb544093740 0x7fb544089270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:02.128 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.121+0000 7fb5437fe640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb544094060 0x7fb5440897b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:02.128 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.121+0000 7fb548c66640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb54412ece0 0x7fb544089cf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:02.128 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.121+0000 7fb5437fe640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb544094060 0x7fb5440897b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:40420/0 (socket says 172.21.15.105:40420) 2024-09-21T01:45:02.129 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.121+0000 7fb543fff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb544093740 0x7fb544089270 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:59856/0 (socket says 172.21.15.105:59856) 2024-09-21T01:45:02.129 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.122+0000 7fb5437fe640 1 -- 172.21.15.105:0/3879992560 learned_addr learned my addr 172.21.15.105:0/3879992560 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:45:02.129 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.122+0000 7fb5437fe640 1 -- 172.21.15.105:0/3879992560 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb54412ece0 msgr2=0x7fb544089cf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:02.129 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.122+0000 7fb5437fe640 1 --2- 172.21.15.105:0/3879992560 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb54412ece0 0x7fb544089cf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.129 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.122+0000 7fb5437fe640 1 -- 172.21.15.105:0/3879992560 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb544093740 msgr2=0x7fb544089270 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:02.129 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.122+0000 7fb5437fe640 1 --2- 172.21.15.105:0/3879992560 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb544093740 0x7fb544089270 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.129 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.122+0000 7fb5437fe640 1 -- 172.21.15.105:0/3879992560 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb534009490 con 0x7fb544094060 2024-09-21T01:45:02.130 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.122+0000 7fb548c66640 1 --2- 172.21.15.105:0/3879992560 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb54412ece0 0x7fb544089cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:45:02.130 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.122+0000 7fb543fff640 1 --2- 172.21.15.105:0/3879992560 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb544093740 0x7fb544089270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:45:02.130 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.122+0000 7fb5437fe640 1 --2- 172.21.15.105:0/3879992560 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb544094060 0x7fb5440897b0 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7fb53800c460 tx=0x7fb53800c920 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:45:02.130 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.122+0000 7fb540ff9640 1 -- 172.21.15.105:0/3879992560 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb538005480 con 0x7fb544094060 2024-09-21T01:45:02.130 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.122+0000 7fb540ff9640 1 -- 172.21.15.105:0/3879992560 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7fb5380035f0 con 0x7fb544094060 2024-09-21T01:45:02.130 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.122+0000 7fb540ff9640 1 -- 172.21.15.105:0/3879992560 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7fb538003a00 con 0x7fb544094060 2024-09-21T01:45:02.131 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.122+0000 7fb54a6c9640 1 -- 172.21.15.105:0/3879992560 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb53c009840 con 0x7fb544094060 2024-09-21T01:45:02.131 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.122+0000 7fb54a6c9640 1 -- 172.21.15.105:0/3879992560 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb54408a230 con 0x7fb544094060 2024-09-21T01:45:02.131 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.123+0000 7fb540ff9640 1 -- 172.21.15.105:0/3879992560 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7fb538003b60 con 0x7fb544094060 2024-09-21T01:45:02.131 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.123+0000 7fb54a6c9640 1 -- 172.21.15.105:0/3879992560 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb5440722c0 con 0x7fb544094060 2024-09-21T01:45:02.131 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.124+0000 7fb540ff9640 1 -- 172.21.15.105:0/3879992560 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(18..18 src has 1..18) v4 ==== 2840+0+0 (secure 0 0 0) 0x7fb53805c380 con 0x7fb544094060 2024-09-21T01:45:02.132 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.127+0000 7fb540ff9640 1 -- 172.21.15.105:0/3879992560 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7fb53803a0b0 con 0x7fb544094060 2024-09-21T01:45:02.258 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.258+0000 7fb54a6c9640 1 -- 172.21.15.105:0/3879992560 --> v1:172.21.15.105:6816/3933313069 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7fb544093b20 con 0x7fb530047560 2024-09-21T01:45:02.259 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.259+0000 7fb540ff9640 1 -- 172.21.15.105:0/3879992560 <== mgr.4112 v1:172.21.15.105:6816/3933313069 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+34967 (unknown 2965378022 0 4199764405) 0x7fb544093b20 con 0x7fb530047560 2024-09-21T01:45:02.259 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:45:02.260 INFO:teuthology.orchestra.run.smithi105.stderr:dumped all 2024-09-21T01:45:02.261 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.260+0000 7fb54a6c9640 1 -- 172.21.15.105:0/3879992560 >> v1:172.21.15.105:6816/3933313069 conn(0x7fb530047560 legacy=0x7fb530049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:02.261 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.260+0000 7fb54a6c9640 1 -- 172.21.15.105:0/3879992560 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb544094060 msgr2=0x7fb5440897b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:02.261 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.260+0000 7fb54a6c9640 1 --2- 172.21.15.105:0/3879992560 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb544094060 0x7fb5440897b0 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7fb53800c460 tx=0x7fb53800c920 comp rx=0 tx=0).stop 2024-09-21T01:45:02.261 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.260+0000 7fb54a6c9640 1 -- 172.21.15.105:0/3879992560 shutdown_connections 2024-09-21T01:45:02.261 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.260+0000 7fb54a6c9640 1 --2- 172.21.15.105:0/3879992560 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb54412ece0 0x7fb544089cf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.262 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.260+0000 7fb54a6c9640 1 --2- 172.21.15.105:0/3879992560 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb544094060 0x7fb5440897b0 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.262 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.260+0000 7fb54a6c9640 1 --2- 172.21.15.105:0/3879992560 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb544093740 0x7fb544089270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.262 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.260+0000 7fb54a6c9640 1 -- 172.21.15.105:0/3879992560 >> 172.21.15.105:0/3879992560 conn(0x7fb544098760 msgr2=0x7fb54408d220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:02.262 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.261+0000 7fb54a6c9640 1 -- 172.21.15.105:0/3879992560 shutdown_connections 2024-09-21T01:45:02.262 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.261+0000 7fb54a6c9640 1 -- 172.21.15.105:0/3879992560 wait complete. 2024-09-21T01:45:02.272 INFO:teuthology.orchestra.run.smithi105.stdout:{"pg_ready":true,"pg_map":{"version":21,"stamp":"2024-09-21T01:45:02.068214+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":109840,"kb_used_data":1744,"kb_used_omap":6,"kb_used_meta":108025,"kb_avail":377377520,"statfs":{"total":386547056640,"available":386434580480,"internally_reserved":0,"allocated":1785856,"data_stored":1345730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6374,"internal_metadata":110618394},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.554763"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T01:44:57.534395+0000","last_change":"2024-09-21T01:44:57.534395+0000","last_active":"2024-09-21T01:44:57.534395+0000","last_peered":"2024-09-21T01:44:57.534395+0000","last_clean":"2024-09-21T01:44:57.534395+0000","last_became_active":"2024-09-21T01:44:55.526300+0000","last_became_peered":"2024-09-21T01:44:55.526300+0000","last_unstale":"2024-09-21T01:44:57.534395+0000","last_undegraded":"2024-09-21T01:44:57.534395+0000","last_fullsized":"2024-09-21T01:44:57.534395+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:38.742537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040868700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T01:44:57.531141+0000","last_change":"2024-09-21T01:44:57.531141+0000","last_active":"2024-09-21T01:44:57.531141+0000","last_peered":"2024-09-21T01:44:57.531141+0000","last_clean":"2024-09-21T01:44:57.531141+0000","last_became_active":"2024-09-21T01:44:55.527636+0000","last_became_peered":"2024-09-21T01:44:55.527636+0000","last_unstale":"2024-09-21T01:44:57.531141+0000","last_undegraded":"2024-09-21T01:44:57.531141+0000","last_fullsized":"2024-09-21T01:44:57.531141+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:57:35.618887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055542700000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T01:44:57.535152+0000","last_change":"2024-09-21T01:44:57.535152+0000","last_active":"2024-09-21T01:44:57.535152+0000","last_peered":"2024-09-21T01:44:57.535152+0000","last_clean":"2024-09-21T01:44:57.535152+0000","last_became_active":"2024-09-21T01:44:55.526163+0000","last_became_peered":"2024-09-21T01:44:55.526163+0000","last_unstale":"2024-09-21T01:44:57.535152+0000","last_undegraded":"2024-09-21T01:44:57.535152+0000","last_fullsized":"2024-09-21T01:44:57.535152+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:57:35.618887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00119484,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T01:44:57.531202+0000","last_change":"2024-09-21T01:44:57.531202+0000","last_active":"2024-09-21T01:44:57.531202+0000","last_peered":"2024-09-21T01:44:57.531202+0000","last_clean":"2024-09-21T01:44:57.531202+0000","last_became_active":"2024-09-21T01:44:55.526110+0000","last_became_peered":"2024-09-21T01:44:55.526110+0000","last_unstale":"2024-09-21T01:44:57.531202+0000","last_undegraded":"2024-09-21T01:44:57.531202+0000","last_fullsized":"2024-09-21T01:44:57.531202+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:38.742537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063215099999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"17'2","reported_seq":17,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T01:44:57.535975+0000","last_change":"2024-09-21T01:44:57.535093+0000","last_active":"2024-09-21T01:44:57.535975+0000","last_peered":"2024-09-21T01:44:57.535975+0000","last_clean":"2024-09-21T01:44:57.535975+0000","last_became_active":"2024-09-21T01:44:55.529008+0000","last_became_peered":"2024-09-21T01:44:55.529008+0000","last_unstale":"2024-09-21T01:44:57.535975+0000","last_undegraded":"2024-09-21T01:44:57.535975+0000","last_fullsized":"2024-09-21T01:44:57.535975+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:43.118995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010245149999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T01:44:57.531535+0000","last_change":"2024-09-21T01:44:57.531535+0000","last_active":"2024-09-21T01:44:57.531535+0000","last_peered":"2024-09-21T01:44:57.531535+0000","last_clean":"2024-09-21T01:44:57.531535+0000","last_became_active":"2024-09-21T01:44:55.526110+0000","last_became_peered":"2024-09-21T01:44:55.526110+0000","last_unstale":"2024-09-21T01:44:57.531535+0000","last_undegraded":"2024-09-21T01:44:57.531535+0000","last_fullsized":"2024-09-21T01:44:57.531535+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T05:54:08.666566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080666500000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":15,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T01:44:57.534522+0000","last_change":"2024-09-21T01:44:57.534522+0000","last_active":"2024-09-21T01:44:57.534522+0000","last_peered":"2024-09-21T01:44:57.534522+0000","last_clean":"2024-09-21T01:44:57.534522+0000","last_became_active":"2024-09-21T01:44:55.528489+0000","last_became_peered":"2024-09-21T01:44:55.528489+0000","last_unstale":"2024-09-21T01:44:57.534522+0000","last_undegraded":"2024-09-21T01:44:57.534522+0000","last_fullsized":"2024-09-21T01:44:57.534522+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T07:40:38.634948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058759600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"15'1","reported_seq":16,"reported_epoch":17,"state":"active+clean","last_fresh":"2024-09-21T01:44:57.534320+0000","last_change":"2024-09-21T01:44:57.534320+0000","last_active":"2024-09-21T01:44:57.534320+0000","last_peered":"2024-09-21T01:44:57.534320+0000","last_clean":"2024-09-21T01:44:57.534320+0000","last_became_active":"2024-09-21T01:44:55.525577+0000","last_became_peered":"2024-09-21T01:44:55.525577+0000","last_unstale":"2024-09-21T01:44:57.534320+0000","last_undegraded":"2024-09-21T01:44:57.534320+0000","last_fullsized":"2024-09-21T01:44:57.534320+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T03:49:16.598027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036256899999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"12'97","reported_seq":141,"reported_epoch":16,"state":"active+clean","last_fresh":"2024-09-21T01:44:56.532376+0000","last_change":"2024-09-21T01:44:51.556547+0000","last_active":"2024-09-21T01:44:56.532376+0000","last_peered":"2024-09-21T01:44:56.532376+0000","last_clean":"2024-09-21T01:44:56.532376+0000","last_became_active":"2024-09-21T01:44:51.555535+0000","last_became_peered":"2024-09-21T01:44:51.555535+0000","last_unstale":"2024-09-21T01:44:56.532376+0000","last_undegraded":"2024-09-21T01:44:56.532376+0000","last_fullsized":"2024-09-21T01:44:56.532376+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:50.489745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:50.489745+0000","last_clean_scrub_stamp":"2024-09-21T01:44:50.489745+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T03:15:49.853900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":9,"seq":38654705669,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27172,"kb_used_data":148,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344668,"statfs":{"total":96636764160,"available":96608940032,"internally_reserved":0,"allocated":151552,"data_stored":41258,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34300000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28299999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36099999999999999}]}]},{"osd":2,"up_from":9,"seq":38654705669,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27168,"kb_used_data":144,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344672,"statfs":{"total":96636764160,"available":96608944128,"internally_reserved":0,"allocated":147456,"data_stored":41239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.313}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41099999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35899999999999999}]}]},{"osd":1,"up_from":9,"seq":38654705669,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27752,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344088,"statfs":{"total":96636764160,"available":96608346112,"internally_reserved":0,"allocated":745472,"data_stored":631626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42399999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32500000000000001}]}]},{"osd":0,"up_from":9,"seq":38654705670,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27748,"kb_used_data":724,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344092,"statfs":{"total":96636764160,"available":96608350208,"internally_reserved":0,"allocated":741376,"data_stored":631607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33300000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T01:45:02.272 INFO:tasks.ceph.ceph_manager.ceph:clean! 2024-09-21T01:45:02.272 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T01:45:02.273 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy 2024-09-21T01:45:02.273 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph health --format=json 2024-09-21T01:45:02.372 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.371+0000 7f96f0c53640 1 -- 172.21.15.105:0/211834674 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f96ec094230 msgr2=0x7f96ec12e5d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:02.372 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.371+0000 7f96f0c53640 1 --2- 172.21.15.105:0/211834674 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f96ec094230 0x7f96ec12e5d0 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f96d4009830 tx=0x7f96d402a3f0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.372 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.371+0000 7f96f0c53640 1 -- 172.21.15.105:0/211834674 shutdown_connections 2024-09-21T01:45:02.372 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.371+0000 7f96f0c53640 1 --2- 172.21.15.105:0/211834674 >> v2:172.21.15.105:3300/0 conn(0x7f96ec12eb10 0x7f96ec130f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.372 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.371+0000 7f96f0c53640 1 --2- 172.21.15.105:0/211834674 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f96ec094230 0x7f96ec12e5d0 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.372 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.371+0000 7f96f0c53640 1 -- 172.21.15.105:0/211834674 >> 172.21.15.105:0/211834674 conn(0x7f96ec098760 msgr2=0x7f96ec098b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:02.372 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.371+0000 7f96f0c53640 1 -- 172.21.15.105:0/211834674 shutdown_connections 2024-09-21T01:45:02.372 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.371+0000 7f96f0c53640 1 -- 172.21.15.105:0/211834674 wait complete. 2024-09-21T01:45:02.375 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.372+0000 7f96f0c53640 1 Processor -- start 2024-09-21T01:45:02.375 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.372+0000 7f96f0c53640 1 -- start start 2024-09-21T01:45:02.375 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.372+0000 7f96f0c53640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f96ec093740 0x7f96ec092630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:02.375 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.372+0000 7f96f0c53640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f96ec094230 0x7f96ec092b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:02.375 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.372+0000 7f96f0c53640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f96ec12eb10 0x7f96ec0930b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:02.375 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.372+0000 7f96f0c53640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f96ec0721a0 con 0x7f96ec12eb10 2024-09-21T01:45:02.375 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.372+0000 7f96f0c53640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f96ec120180 con 0x7f96ec094230 2024-09-21T01:45:02.375 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.372+0000 7f96f0c53640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f96ec071be0 con 0x7f96ec093740 2024-09-21T01:45:02.375 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.373+0000 7f96e9d9b640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f96ec094230 0x7f96ec092b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:02.375 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.373+0000 7f96ead9d640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f96ec12eb10 0x7f96ec0930b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:02.376 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.373+0000 7f96ea59c640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f96ec093740 0x7f96ec092630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:02.376 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.373+0000 7f96ea59c640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f96ec093740 0x7f96ec092630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:38662/0 (socket says 172.21.15.105:38662) 2024-09-21T01:45:02.376 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.373+0000 7f96ead9d640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f96ec12eb10 0x7f96ec0930b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:40430/0 (socket says 172.21.15.105:40430) 2024-09-21T01:45:02.376 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.373+0000 7f96e9d9b640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f96ec094230 0x7f96ec092b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:59874/0 (socket says 172.21.15.105:59874) 2024-09-21T01:45:02.376 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.373+0000 7f96ea59c640 1 -- 172.21.15.105:0/2886743097 learned_addr learned my addr 172.21.15.105:0/2886743097 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:45:02.376 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.373+0000 7f96ead9d640 1 -- 172.21.15.105:0/2886743097 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f96ec093740 msgr2=0x7f96ec092630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:02.376 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.373+0000 7f96ead9d640 1 --2- 172.21.15.105:0/2886743097 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f96ec093740 0x7f96ec092630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.376 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.373+0000 7f96ead9d640 1 -- 172.21.15.105:0/2886743097 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f96ec094230 msgr2=0x7f96ec092b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:02.376 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.373+0000 7f96ead9d640 1 --2- 172.21.15.105:0/2886743097 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f96ec094230 0x7f96ec092b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.376 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.373+0000 7f96ead9d640 1 -- 172.21.15.105:0/2886743097 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f96e4009840 con 0x7f96ec12eb10 2024-09-21T01:45:02.376 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.373+0000 7f96e9d9b640 1 --2- 172.21.15.105:0/2886743097 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f96ec094230 0x7f96ec092b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:45:02.376 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.373+0000 7f96ead9d640 1 --2- 172.21.15.105:0/2886743097 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f96ec12eb10 0x7f96ec0930b0 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f96e400ec50 tx=0x7f96e400c6a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:45:02.376 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.373+0000 7f96e2ffd640 1 -- 172.21.15.105:0/2886743097 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f96e4003b70 con 0x7f96ec12eb10 2024-09-21T01:45:02.376 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.373+0000 7f96e2ffd640 1 -- 172.21.15.105:0/2886743097 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f96e4003cd0 con 0x7f96ec12eb10 2024-09-21T01:45:02.377 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.373+0000 7f96e2ffd640 1 -- 172.21.15.105:0/2886743097 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f96e4004280 con 0x7f96ec12eb10 2024-09-21T01:45:02.377 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.373+0000 7f96f0c53640 1 -- 172.21.15.105:0/2886743097 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f96d4009490 con 0x7f96ec12eb10 2024-09-21T01:45:02.377 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.373+0000 7f96f0c53640 1 -- 172.21.15.105:0/2886743097 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f96ec11f8d0 con 0x7f96ec12eb10 2024-09-21T01:45:02.377 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.374+0000 7f96e2ffd640 1 -- 172.21.15.105:0/2886743097 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f96e40043e0 con 0x7f96ec12eb10 2024-09-21T01:45:02.377 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.374+0000 7f96f0c53640 1 -- 172.21.15.105:0/2886743097 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f96ec1302a0 con 0x7f96ec12eb10 2024-09-21T01:45:02.377 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.375+0000 7f96e2ffd640 1 -- 172.21.15.105:0/2886743097 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(18..18 src has 1..18) v4 ==== 2840+0+0 (secure 0 0 0) 0x7f96e4014070 con 0x7f96ec12eb10 2024-09-21T01:45:02.378 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.378+0000 7f96e2ffd640 1 -- 172.21.15.105:0/2886743097 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f96e403b230 con 0x7f96ec12eb10 2024-09-21T01:45:02.534 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.534+0000 7f96f0c53640 1 -- 172.21.15.105:0/2886743097 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "health", "format": "json"} v 0) v1 -- 0x7f96ec093b20 con 0x7f96ec12eb10 2024-09-21T01:45:02.534 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.534+0000 7f96e2ffd640 1 -- 172.21.15.105:0/2886743097 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "health", "format": "json"}]=0 v0) v1 ==== 72+0+46 (secure 0 0 0) 0x7f96e4003e80 con 0x7f96ec12eb10 2024-09-21T01:45:02.535 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:45:02.535 INFO:teuthology.orchestra.run.smithi105.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T01:45:02.536 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.535+0000 7f96f0c53640 1 -- 172.21.15.105:0/2886743097 >> v1:172.21.15.105:6816/3933313069 conn(0x7f96cc047560 legacy=0x7f96cc049a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:02.536 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.535+0000 7f96f0c53640 1 -- 172.21.15.105:0/2886743097 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f96ec12eb10 msgr2=0x7f96ec0930b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:02.536 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.535+0000 7f96f0c53640 1 --2- 172.21.15.105:0/2886743097 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f96ec12eb10 0x7f96ec0930b0 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f96e400ec50 tx=0x7f96e400c6a0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.536 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.536+0000 7f96f0c53640 1 -- 172.21.15.105:0/2886743097 shutdown_connections 2024-09-21T01:45:02.537 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.536+0000 7f96f0c53640 1 --2- 172.21.15.105:0/2886743097 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f96ec12eb10 0x7f96ec0930b0 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.537 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.536+0000 7f96f0c53640 1 --2- 172.21.15.105:0/2886743097 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f96ec094230 0x7f96ec092b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.537 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.536+0000 7f96f0c53640 1 --2- 172.21.15.105:0/2886743097 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f96ec093740 0x7f96ec092630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.537 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.536+0000 7f96f0c53640 1 -- 172.21.15.105:0/2886743097 >> 172.21.15.105:0/2886743097 conn(0x7f96ec098760 msgr2=0x7f96ec094e40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:02.537 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.536+0000 7f96f0c53640 1 -- 172.21.15.105:0/2886743097 shutdown_connections 2024-09-21T01:45:02.537 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.536+0000 7f96f0c53640 1 -- 172.21.15.105:0/2886743097 wait complete. 2024-09-21T01:45:02.547 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy done 2024-09-21T01:45:02.547 INFO:teuthology.run_tasks:Running task exec... 2024-09-21T01:45:02.562 INFO:teuthology.task.exec:Executing custom commands... 2024-09-21T01:45:02.563 INFO:teuthology.task.exec:Running commands on role osd.0 host ubuntu@smithi105.front.sepia.ceph.com 2024-09-21T01:45:02.563 DEBUG:teuthology.orchestra.run.smithi105:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph osd set-require-min-compat-client reef' 2024-09-21T01:45:02.661 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.661+0000 7f9a11983640 1 -- 172.21.15.105:0/58122875 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f9a0c131690 msgr2=0x7f9a0c133a80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:02.662 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.661+0000 7f9a11983640 1 --2- 172.21.15.105:0/58122875 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f9a0c131690 0x7f9a0c133a80 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f9a04009be0 tx=0x7f9a0402a770 comp rx=0 tx=0).stop 2024-09-21T01:45:02.662 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.661+0000 7f9a11983640 1 -- 172.21.15.105:0/58122875 shutdown_connections 2024-09-21T01:45:02.662 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.661+0000 7f9a11983640 1 --2- 172.21.15.105:0/58122875 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f9a0c131690 0x7f9a0c133a80 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.662 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.661+0000 7f9a11983640 1 --2- 172.21.15.105:0/58122875 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9a0c12ca50 0x7f9a0c131150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.662 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.661+0000 7f9a11983640 1 -- 172.21.15.105:0/58122875 >> 172.21.15.105:0/58122875 conn(0x7f9a0c09cd00 msgr2=0x7f9a0c09d110 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:02.662 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.661+0000 7f9a11983640 1 -- 172.21.15.105:0/58122875 shutdown_connections 2024-09-21T01:45:02.662 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.661+0000 7f9a11983640 1 -- 172.21.15.105:0/58122875 wait complete. 2024-09-21T01:45:02.664 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.662+0000 7f9a11983640 1 Processor -- start 2024-09-21T01:45:02.664 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.662+0000 7f9a11983640 1 -- start start 2024-09-21T01:45:02.664 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.662+0000 7f9a11983640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9a0c06ff30 0x7f9a0c12d150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:02.664 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.662+0000 7f9a11983640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9a0c12ca50 0x7f9a0c12d690 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:02.664 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.662+0000 7f9a11983640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f9a0c131690 0x7f9a0c12dbd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:02.664 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.662+0000 7f9a11983640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9a0c1309c0 con 0x7f9a0c06ff30 2024-09-21T01:45:02.664 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.662+0000 7f9a11983640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f9a0c12f350 con 0x7f9a0c12ca50 2024-09-21T01:45:02.665 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.662+0000 7f9a11983640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f9a0c130700 con 0x7f9a0c131690 2024-09-21T01:45:02.665 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.663+0000 7f9a0affd640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9a0c06ff30 0x7f9a0c12d150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:02.665 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.663+0000 7f9a027fc640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9a0c12ca50 0x7f9a0c12d690 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:02.665 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.663+0000 7f9a0affd640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9a0c06ff30 0x7f9a0c12d150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:40444/0 (socket says 172.21.15.105:40444) 2024-09-21T01:45:02.665 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.663+0000 7f9a0affd640 1 -- 172.21.15.105:0/2389724382 learned_addr learned my addr 172.21.15.105:0/2389724382 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:45:02.665 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.663+0000 7f9a027fc640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9a0c12ca50 0x7f9a0c12d690 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:59884/0 (socket says 172.21.15.105:59884) 2024-09-21T01:45:02.665 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.663+0000 7f9a0b7fe640 1 --2- 172.21.15.105:0/2389724382 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f9a0c131690 0x7f9a0c12dbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:02.665 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.663+0000 7f9a0affd640 1 -- 172.21.15.105:0/2389724382 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f9a0c131690 msgr2=0x7f9a0c12dbd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:02.665 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.663+0000 7f9a0affd640 1 --2- 172.21.15.105:0/2389724382 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f9a0c131690 0x7f9a0c12dbd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.665 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.663+0000 7f9a0affd640 1 -- 172.21.15.105:0/2389724382 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9a0c12ca50 msgr2=0x7f9a0c12d690 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:02.665 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.663+0000 7f9a0affd640 1 --2- 172.21.15.105:0/2389724382 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9a0c12ca50 0x7f9a0c12d690 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:02.665 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.663+0000 7f9a0affd640 1 -- 172.21.15.105:0/2389724382 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f99f4009490 con 0x7f9a0c06ff30 2024-09-21T01:45:02.665 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.663+0000 7f9a027fc640 1 --2- 172.21.15.105:0/2389724382 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9a0c12ca50 0x7f9a0c12d690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:45:02.665 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.663+0000 7f9a0affd640 1 --2- 172.21.15.105:0/2389724382 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9a0c06ff30 0x7f9a0c12d150 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f99fc00c700 tx=0x7f99fc00cbc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:45:02.666 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.663+0000 7f9a08ff9640 1 -- 172.21.15.105:0/2389724382 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f99fc00a840 con 0x7f9a0c06ff30 2024-09-21T01:45:02.666 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.663+0000 7f9a08ff9640 1 -- 172.21.15.105:0/2389724382 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f99fc00aa90 con 0x7f9a0c06ff30 2024-09-21T01:45:02.666 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.663+0000 7f9a11983640 1 -- 172.21.15.105:0/2389724382 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9a04009840 con 0x7f9a0c06ff30 2024-09-21T01:45:02.666 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.663+0000 7f9a08ff9640 1 -- 172.21.15.105:0/2389724382 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f99fc012420 con 0x7f9a0c06ff30 2024-09-21T01:45:02.666 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.663+0000 7f9a11983640 1 -- 172.21.15.105:0/2389724382 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9a0c12e1c0 con 0x7f9a0c06ff30 2024-09-21T01:45:02.666 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.664+0000 7f9a08ff9640 1 -- 172.21.15.105:0/2389724382 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f99fc012580 con 0x7f9a0c06ff30 2024-09-21T01:45:02.666 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.664+0000 7f9a11983640 1 -- 172.21.15.105:0/2389724382 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9a0c12c340 con 0x7f9a0c06ff30 2024-09-21T01:45:02.667 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.664+0000 7f9a08ff9640 1 -- 172.21.15.105:0/2389724382 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(18..18 src has 1..18) v4 ==== 2840+0+0 (secure 0 0 0) 0x7f99fc05c460 con 0x7f9a0c06ff30 2024-09-21T01:45:02.668 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.668+0000 7f9a08ff9640 1 -- 172.21.15.105:0/2389724382 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f99fc03a150 con 0x7f9a0c06ff30 2024-09-21T01:45:02.810 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:02.810+0000 7f9a11983640 1 -- 172.21.15.105:0/2389724382 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd set-require-min-compat-client", "version": "reef"} v 0) v1 -- 0x7f9a0c12c100 con 0x7f9a0c06ff30 2024-09-21T01:45:03.552 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.552+0000 7f9a08ff9640 1 -- 172.21.15.105:0/2389724382 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd set-require-min-compat-client", "version": "reef"}]=0 set require_min_compat_client to reef v19) v1 ==== 137+0+0 (secure 0 0 0) 0x7f9a0c12c100 con 0x7f9a0c06ff30 2024-09-21T01:45:03.553 INFO:teuthology.orchestra.run.smithi105.stderr:set require_min_compat_client to reef 2024-09-21T01:45:03.554 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.554+0000 7f9a11983640 1 -- 172.21.15.105:0/2389724382 >> v1:172.21.15.105:6816/3933313069 conn(0x7f99ec047590 legacy=0x7f99ec049a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:03.554 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.554+0000 7f9a11983640 1 -- 172.21.15.105:0/2389724382 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9a0c06ff30 msgr2=0x7f9a0c12d150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:03.555 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.554+0000 7f9a11983640 1 --2- 172.21.15.105:0/2389724382 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9a0c06ff30 0x7f9a0c12d150 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f99fc00c700 tx=0x7f99fc00cbc0 comp rx=0 tx=0).stop 2024-09-21T01:45:03.555 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.554+0000 7f9a11983640 1 -- 172.21.15.105:0/2389724382 shutdown_connections 2024-09-21T01:45:03.555 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.554+0000 7f9a11983640 1 --2- 172.21.15.105:0/2389724382 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f9a0c131690 0x7f9a0c12dbd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:03.555 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.554+0000 7f9a11983640 1 --2- 172.21.15.105:0/2389724382 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9a0c12ca50 0x7f9a0c12d690 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:03.555 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.554+0000 7f9a11983640 1 --2- 172.21.15.105:0/2389724382 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9a0c06ff30 0x7f9a0c12d150 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:03.555 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.554+0000 7f9a11983640 1 -- 172.21.15.105:0/2389724382 >> 172.21.15.105:0/2389724382 conn(0x7f9a0c09cd00 msgr2=0x7f9a0c12aaf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:03.555 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.554+0000 7f9a11983640 1 -- 172.21.15.105:0/2389724382 shutdown_connections 2024-09-21T01:45:03.555 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.554+0000 7f9a11983640 1 -- 172.21.15.105:0/2389724382 wait complete. 2024-09-21T01:45:03.566 INFO:teuthology.run_tasks:Running task print... 2024-09-21T01:45:03.574 INFO:teuthology.task.print:**** done ceph 2024-09-21T01:45:03.574 INFO:teuthology.run_tasks:Running task print... 2024-09-21T01:45:03.640 INFO:teuthology.task.print:*** upgrading, no cephfs present 2024-09-21T01:45:03.641 INFO:teuthology.run_tasks:Running task exec... 2024-09-21T01:45:03.779 INFO:teuthology.task.exec:Executing custom commands... 2024-09-21T01:45:03.779 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi105.front.sepia.ceph.com 2024-09-21T01:45:03.779 DEBUG:teuthology.orchestra.run.smithi105:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs dump' 2024-09-21T01:45:03.881 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.881+0000 7f0f3c879640 1 -- 172.21.15.105:0/4041448379 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0f38129610 msgr2=0x7f0f381313e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:03.882 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.881+0000 7f0f3c879640 1 --2- 172.21.15.105:0/4041448379 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0f38129610 0x7f0f381313e0 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f0f2c009be0 tx=0x7f0f2c02a7c0 comp rx=0 tx=0).stop 2024-09-21T01:45:03.882 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.881+0000 7f0f3c879640 1 -- 172.21.15.105:0/4041448379 shutdown_connections 2024-09-21T01:45:03.882 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.881+0000 7f0f3c879640 1 --2- 172.21.15.105:0/4041448379 >> v2:172.21.15.105:3300/0 conn(0x7f0f38131920 0x7f0f38133d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:03.882 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.881+0000 7f0f3c879640 1 --2- 172.21.15.105:0/4041448379 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0f38129610 0x7f0f381313e0 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:03.882 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.881+0000 7f0f3c879640 1 -- 172.21.15.105:0/4041448379 >> 172.21.15.105:0/4041448379 conn(0x7f0f3809ca50 msgr2=0x7f0f3809ce60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:03.882 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.881+0000 7f0f3c879640 1 -- 172.21.15.105:0/4041448379 shutdown_connections 2024-09-21T01:45:03.882 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.881+0000 7f0f3c879640 1 -- 172.21.15.105:0/4041448379 wait complete. 2024-09-21T01:45:03.884 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.882+0000 7f0f3c879640 1 Processor -- start 2024-09-21T01:45:03.884 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.882+0000 7f0f3c879640 1 -- start start 2024-09-21T01:45:03.884 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.882+0000 7f0f3c879640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0f38128b20 0x7f0f3812ce80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:03.884 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.882+0000 7f0f3c879640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0f38129610 0x7f0f3812d3c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:03.884 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.882+0000 7f0f3c879640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0f38131920 0x7f0f3812d900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:45:03.884 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.882+0000 7f0f3c879640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0f3812fb00 con 0x7f0f38129610 2024-09-21T01:45:03.885 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.882+0000 7f0f3c879640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 v1 -- 0x7f0f38126880 con 0x7f0f38128b20 2024-09-21T01:45:03.885 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.882+0000 7f0f3c879640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 v1 -- 0x7f0f3812ee90 con 0x7f0f38131920 2024-09-21T01:45:03.885 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.882+0000 7f0f35d9b640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0f38129610 0x7f0f3812d3c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:03.885 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.882+0000 7f0f3659c640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0f38128b20 0x7f0f3812ce80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:03.885 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.882+0000 7f0f35d9b640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0f38129610 0x7f0f3812d3c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:40458/0 (socket says 172.21.15.105:40458) 2024-09-21T01:45:03.885 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.882+0000 7f0f35d9b640 1 -- 172.21.15.105:0/1494749719 learned_addr learned my addr 172.21.15.105:0/1494749719 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:45:03.885 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.882+0000 7f0f36d9d640 1 --2- 172.21.15.105:0/1494749719 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0f38131920 0x7f0f3812d900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:45:03.885 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.883+0000 7f0f35d9b640 1 -- 172.21.15.105:0/1494749719 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0f38131920 msgr2=0x7f0f3812d900 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:03.885 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.883+0000 7f0f35d9b640 1 --2- 172.21.15.105:0/1494749719 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0f38131920 0x7f0f3812d900 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:03.885 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.883+0000 7f0f35d9b640 1 -- 172.21.15.105:0/1494749719 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0f38128b20 msgr2=0x7f0f3812ce80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:03.885 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.883+0000 7f0f35d9b640 1 --2- 172.21.15.105:0/1494749719 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0f38128b20 0x7f0f3812ce80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:03.885 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.883+0000 7f0f35d9b640 1 -- 172.21.15.105:0/1494749719 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0f28009490 con 0x7f0f38129610 2024-09-21T01:45:03.885 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.883+0000 7f0f3659c640 1 --2- 172.21.15.105:0/1494749719 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0f38128b20 0x7f0f3812ce80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:45:03.885 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.883+0000 7f0f36d9d640 1 --2- 172.21.15.105:0/1494749719 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0f38131920 0x7f0f3812d900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:45:03.886 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.883+0000 7f0f35d9b640 1 --2- 172.21.15.105:0/1494749719 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0f38129610 0x7f0f3812d3c0 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f0f2c009b30 tx=0x7f0f2c003c90 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:45:03.886 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.883+0000 7f0f26ffd640 1 -- 172.21.15.105:0/1494749719 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0f2c004450 con 0x7f0f38129610 2024-09-21T01:45:03.886 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.883+0000 7f0f26ffd640 1 -- 172.21.15.105:0/1494749719 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) v1 ==== 141+0+0 (secure 0 0 0) 0x7f0f2c0046a0 con 0x7f0f38129610 2024-09-21T01:45:03.886 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.883+0000 7f0f3c879640 1 -- 172.21.15.105:0/1494749719 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0f2c009840 con 0x7f0f38129610 2024-09-21T01:45:03.886 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.883+0000 7f0f26ffd640 1 -- 172.21.15.105:0/1494749719 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 v1 ==== 413+0+0 (secure 0 0 0) 0x7f0f2c03c680 con 0x7f0f38129610 2024-09-21T01:45:03.886 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.883+0000 7f0f3c879640 1 -- 172.21.15.105:0/1494749719 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0f3812e5d0 con 0x7f0f38129610 2024-09-21T01:45:03.886 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.884+0000 7f0f3c879640 1 -- 172.21.15.105:0/1494749719 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0f38128940 con 0x7f0f38129610 2024-09-21T01:45:03.887 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.884+0000 7f0f26ffd640 1 -- 172.21.15.105:0/1494749719 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 5) v1 ==== 55212+0+0 (secure 0 0 0) 0x7f0f2c03c7e0 con 0x7f0f38129610 2024-09-21T01:45:03.887 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.885+0000 7f0f26ffd640 1 -- 172.21.15.105:0/1494749719 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(19..19 src has 1..19) v4 ==== 2812+0+0 (secure 0 0 0) 0x7f0f2c07e360 con 0x7f0f38129610 2024-09-21T01:45:03.888 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:03.888+0000 7f0f26ffd640 1 -- 172.21.15.105:0/1494749719 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+146310 (secure 0 0 0) 0x7f0f2c05fa80 con 0x7f0f38129610 2024-09-21T01:45:04.044 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:04.044+0000 7f0f3c879640 1 -- 172.21.15.105:0/1494749719 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "fs dump"} v 0) v1 -- 0x7f0f38128f00 con 0x7f0f38129610 2024-09-21T01:45:04.044 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:04.044+0000 7f0f26ffd640 1 -- 172.21.15.105:0/1494749719 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump"}]=0 dumped fsmap epoch 1 v1) v1 ==== 75+0+358 (secure 0 0 0) 0x7f0f2c05f6c0 con 0x7f0f38129610 2024-09-21T01:45:04.045 INFO:teuthology.orchestra.run.smithi105.stdout:e1 2024-09-21T01:45:04.045 INFO:teuthology.orchestra.run.smithi105.stdout:enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T01:45:04.045 INFO:teuthology.orchestra.run.smithi105.stdout:default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T01:45:04.045 INFO:teuthology.orchestra.run.smithi105.stdout:legacy client fscid: -1 2024-09-21T01:45:04.045 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:45:04.045 INFO:teuthology.orchestra.run.smithi105.stdout:No filesystems configured 2024-09-21T01:45:04.045 INFO:teuthology.orchestra.run.smithi105.stderr:dumped fsmap epoch 1 2024-09-21T01:45:04.046 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:04.046+0000 7f0f3c879640 1 -- 172.21.15.105:0/1494749719 >> v1:172.21.15.105:6816/3933313069 conn(0x7f0f18047590 legacy=0x7f0f18049a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:04.046 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:04.046+0000 7f0f3c879640 1 -- 172.21.15.105:0/1494749719 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0f38129610 msgr2=0x7f0f3812d3c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:45:04.047 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:04.046+0000 7f0f3c879640 1 --2- 172.21.15.105:0/1494749719 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0f38129610 0x7f0f3812d3c0 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f0f2c009b30 tx=0x7f0f2c003c90 comp rx=0 tx=0).stop 2024-09-21T01:45:04.047 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:04.046+0000 7f0f3c879640 1 -- 172.21.15.105:0/1494749719 shutdown_connections 2024-09-21T01:45:04.047 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:04.046+0000 7f0f3c879640 1 --2- 172.21.15.105:0/1494749719 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0f38131920 0x7f0f3812d900 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:04.047 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:04.046+0000 7f0f3c879640 1 --2- 172.21.15.105:0/1494749719 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0f38129610 0x7f0f3812d3c0 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:04.047 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:04.046+0000 7f0f3c879640 1 --2- 172.21.15.105:0/1494749719 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0f38128b20 0x7f0f3812ce80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:45:04.047 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:04.046+0000 7f0f3c879640 1 -- 172.21.15.105:0/1494749719 >> 172.21.15.105:0/1494749719 conn(0x7f0f3809ca50 msgr2=0x7f0f38128230 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:45:04.047 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:04.046+0000 7f0f3c879640 1 -- 172.21.15.105:0/1494749719 shutdown_connections 2024-09-21T01:45:04.047 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:45:04.046+0000 7f0f3c879640 1 -- 172.21.15.105:0/1494749719 wait complete. 2024-09-21T01:45:04.057 INFO:teuthology.run_tasks:Running task install.upgrade... 2024-09-21T01:45:04.065 INFO:teuthology.task.install:project ceph config {'mon.a': None} overrides {'flavor': 'default', 'sha1': '1843221895adccfb3b692060251dd6a0183bc410'} 2024-09-21T01:45:04.065 INFO:teuthology.task.install:remote ubuntu@smithi105.front.sepia.ceph.com config {'flavor': 'default', 'sha1': '1843221895adccfb3b692060251dd6a0183bc410'} 2024-09-21T01:45:04.066 INFO:teuthology.task.install:extra packages: [] 2024-09-21T01:45:04.070 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-21T01:45:04.070 INFO:teuthology.task.install:Upgrading ceph rpm packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd 2024-09-21T01:45:04.070 DEBUG:teuthology.orchestra.run.smithi105:> rpm -q ceph-common --qf '%{VERSION}-%{RELEASE}' 2024-09-21T01:45:04.091 INFO:teuthology.orchestra.run.smithi105.stdout:18.2.4-871.g6a1bdf85.el9 2024-09-21T01:45:04.091 INFO:teuthology.packaging:The installed version of ceph-common is 18.2.4-871.g6a1bdf85.el9 2024-09-21T01:45:04.092 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=1843221895adccfb3b692060251dd6a0183bc410 2024-09-21T01:45:04.242 INFO:teuthology.task.install:Ceph rpm upgrade from 18.2.4-871.g6a1bdf85.el9 to 19.3.0-5120.g18432218 2024-09-21T01:45:04.242 INFO:teuthology.task.install.rpm:Host smithi105 is: centos 9.stream x86_64 2024-09-21T01:45:04.242 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=1843221895adccfb3b692060251dd6a0183bc410 2024-09-21T01:45:04.379 INFO:teuthology.task.install.rpm:Repo base URL: https://4.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20240920.212106-debug/1843221895adccfb3b692060251dd6a0183bc410/centos/9/flavors/default/ 2024-09-21T01:45:04.380 DEBUG:teuthology.orchestra.run.smithi105:> sudo rm -f /etc/yum.repos.d/ceph.repo 2024-09-21T01:45:04.512 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20240920.212106-debug/1843221895adccfb3b692060251dd6a0183bc410/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20240920.212106-debug/1843221895adccfb3b692060251dd6a0183bc410/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20240920.212106-debug/1843221895adccfb3b692060251dd6a0183bc410/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-21T01:45:04.512 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T01:45:04.513 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-21T01:45:04.537 DEBUG:teuthology.orchestra.run.smithi105:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/1843221895adccfb3b692060251dd6a0183bc410/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-21T01:45:04.603 DEBUG:teuthology.orchestra.run.smithi105:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-21T01:45:04.667 DEBUG:teuthology.orchestra.run.smithi105:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-21T01:45:04.723 INFO:teuthology.orchestra.run.smithi105.stdout:check_obsoletes = 1 2024-09-21T01:45:04.734 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum clean all 2024-09-21T01:45:04.963 INFO:teuthology.orchestra.run.smithi105.stdout:60 files removed 2024-09-21T01:45:04.991 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y upgrade ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse ceph-volume librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd 2024-09-21T01:45:05.541 INFO:teuthology.orchestra.run.smithi105.stdout:ceph packages for x86_64 323 kB/s | 84 kB 00:00 2024-09-21T01:45:05.842 INFO:teuthology.orchestra.run.smithi105.stdout:ceph noarch packages 95 kB/s | 24 kB 00:00 2024-09-21T01:45:06.109 INFO:teuthology.orchestra.run.smithi105.stdout:ceph source packages 8.3 kB/s | 1.9 kB 00:00 2024-09-21T01:45:06.718 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - BaseOS 14 MB/s | 8.3 MB 00:00 2024-09-21T01:45:09.303 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - AppStream 14 MB/s | 20 MB 00:01 2024-09-21T01:45:15.571 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - CRB 2.6 MB/s | 6.5 MB 00:02 2024-09-21T01:45:17.088 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - Extras packages 66 kB/s | 19 kB 00:00 2024-09-21T01:45:18.259 INFO:teuthology.orchestra.run.smithi105.stdout:Extra Packages for Enterprise Linux 22 MB/s | 23 MB 00:01 2024-09-21T01:45:23.749 INFO:teuthology.orchestra.run.smithi105.stdout:lab-extras 27 kB/s | 1.7 kB 00:00 2024-09-21T01:45:25.861 INFO:teuthology.orchestra.run.smithi105.stderr:Package cephadm available, but not installed. 2024-09-21T01:45:25.862 INFO:teuthology.orchestra.run.smithi105.stdout:No match for argument: cephadm 2024-09-21T01:45:25.867 INFO:teuthology.orchestra.run.smithi105.stderr:Package ceph-mgr-dashboard available, but not installed. 2024-09-21T01:45:25.867 INFO:teuthology.orchestra.run.smithi105.stdout:No match for argument: ceph-mgr-dashboard 2024-09-21T01:45:25.868 INFO:teuthology.orchestra.run.smithi105.stderr:Package ceph-mgr-diskprediction-local available, but not installed. 2024-09-21T01:45:25.868 INFO:teuthology.orchestra.run.smithi105.stdout:No match for argument: ceph-mgr-diskprediction-local 2024-09-21T01:45:25.868 INFO:teuthology.orchestra.run.smithi105.stderr:Package ceph-mgr-rook available, but not installed. 2024-09-21T01:45:25.868 INFO:teuthology.orchestra.run.smithi105.stdout:No match for argument: ceph-mgr-rook 2024-09-21T01:45:25.869 INFO:teuthology.orchestra.run.smithi105.stderr:Package ceph-mgr-cephadm available, but not installed. 2024-09-21T01:45:25.869 INFO:teuthology.orchestra.run.smithi105.stdout:No match for argument: ceph-mgr-cephadm 2024-09-21T01:45:25.871 INFO:teuthology.orchestra.run.smithi105.stderr:Package ceph-volume available, but not installed. 2024-09-21T01:45:25.871 INFO:teuthology.orchestra.run.smithi105.stdout:No match for argument: ceph-volume 2024-09-21T01:45:25.961 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T01:45:25.964 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:45:25.964 INFO:teuthology.orchestra.run.smithi105.stdout: Package Arch Version Repository Size 2024-09-21T01:45:25.964 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:45:25.964 INFO:teuthology.orchestra.run.smithi105.stdout:Upgrading: 2024-09-21T01:45:25.965 INFO:teuthology.orchestra.run.smithi105.stdout: ceph x86_64 2:19.3.0-5120.g18432218.el9 ceph 6.6 k 2024-09-21T01:45:25.965 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-base x86_64 2:19.3.0-5120.g18432218.el9 ceph 5.5 M 2024-09-21T01:45:25.965 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-common x86_64 2:19.3.0-5120.g18432218.el9 ceph 20 M 2024-09-21T01:45:25.965 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-fuse x86_64 2:19.3.0-5120.g18432218.el9 ceph 873 k 2024-09-21T01:45:25.965 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-immutable-object-cache 2024-09-21T01:45:25.965 INFO:teuthology.orchestra.run.smithi105.stdout: x86_64 2:19.3.0-5120.g18432218.el9 ceph 148 k 2024-09-21T01:45:25.965 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mds x86_64 2:19.3.0-5120.g18432218.el9 ceph 2.3 M 2024-09-21T01:45:25.965 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr x86_64 2:19.3.0-5120.g18432218.el9 ceph 1.5 M 2024-09-21T01:45:25.965 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-modules-core noarch 2:19.3.0-5120.g18432218.el9 ceph-noarch 292 k 2024-09-21T01:45:25.965 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mon x86_64 2:19.3.0-5120.g18432218.el9 ceph 5.0 M 2024-09-21T01:45:25.965 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-osd x86_64 2:19.3.0-5120.g18432218.el9 ceph 18 M 2024-09-21T01:45:25.965 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-radosgw x86_64 2:19.3.0-5120.g18432218.el9 ceph 10 M 2024-09-21T01:45:25.965 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-selinux x86_64 2:19.3.0-5120.g18432218.el9 ceph 25 k 2024-09-21T01:45:25.965 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-test x86_64 2:19.3.0-5120.g18432218.el9 ceph 40 M 2024-09-21T01:45:25.965 INFO:teuthology.orchestra.run.smithi105.stdout: libcephfs-devel x86_64 2:19.3.0-5120.g18432218.el9 ceph 32 k 2024-09-21T01:45:25.966 INFO:teuthology.orchestra.run.smithi105.stdout: libcephfs2 x86_64 2:19.3.0-5120.g18432218.el9 ceph 766 k 2024-09-21T01:45:25.966 INFO:teuthology.orchestra.run.smithi105.stdout: libcephsqlite x86_64 2:19.3.0-5120.g18432218.el9 ceph 158 k 2024-09-21T01:45:25.966 INFO:teuthology.orchestra.run.smithi105.stdout: librados-devel x86_64 2:19.3.0-5120.g18432218.el9 ceph 127 k 2024-09-21T01:45:25.966 INFO:teuthology.orchestra.run.smithi105.stdout: librados2 x86_64 2:19.3.0-5120.g18432218.el9 ceph 3.4 M 2024-09-21T01:45:25.966 INFO:teuthology.orchestra.run.smithi105.stdout: libradosstriper1 x86_64 2:19.3.0-5120.g18432218.el9 ceph 507 k 2024-09-21T01:45:25.966 INFO:teuthology.orchestra.run.smithi105.stdout: librbd1 x86_64 2:19.3.0-5120.g18432218.el9 ceph 3.2 M 2024-09-21T01:45:25.966 INFO:teuthology.orchestra.run.smithi105.stdout: librgw2 x86_64 2:19.3.0-5120.g18432218.el9 ceph 5.3 M 2024-09-21T01:45:25.966 INFO:teuthology.orchestra.run.smithi105.stdout: python3-ceph-argparse x86_64 2:19.3.0-5120.g18432218.el9 ceph 45 k 2024-09-21T01:45:25.966 INFO:teuthology.orchestra.run.smithi105.stdout: python3-ceph-common x86_64 2:19.3.0-5120.g18432218.el9 ceph 148 k 2024-09-21T01:45:25.966 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cephfs x86_64 2:19.3.0-5120.g18432218.el9 ceph 161 k 2024-09-21T01:45:25.966 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rados x86_64 2:19.3.0-5120.g18432218.el9 ceph 321 k 2024-09-21T01:45:25.966 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rbd x86_64 2:19.3.0-5120.g18432218.el9 ceph 307 k 2024-09-21T01:45:25.966 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rgw x86_64 2:19.3.0-5120.g18432218.el9 ceph 99 k 2024-09-21T01:45:25.966 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-fuse x86_64 2:19.3.0-5120.g18432218.el9 ceph 86 k 2024-09-21T01:45:25.967 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-mirror x86_64 2:19.3.0-5120.g18432218.el9 ceph 3.1 M 2024-09-21T01:45:25.967 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-nbd x86_64 2:19.3.0-5120.g18432218.el9 ceph 170 k 2024-09-21T01:45:25.967 INFO:teuthology.orchestra.run.smithi105.stdout:Installing dependencies: 2024-09-21T01:45:25.967 INFO:teuthology.orchestra.run.smithi105.stdout: libnbd x86_64 1.20.2-2.el9 appstream 171 k 2024-09-21T01:45:25.967 INFO:teuthology.orchestra.run.smithi105.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2024-09-21T01:45:25.967 INFO:teuthology.orchestra.run.smithi105.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2024-09-21T01:45:25.967 INFO:teuthology.orchestra.run.smithi105.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2024-09-21T01:45:25.967 INFO:teuthology.orchestra.run.smithi105.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-21T01:45:25.967 INFO:teuthology.orchestra.run.smithi105.stdout: qatlib x86_64 24.02.0-1.el9 appstream 221 k 2024-09-21T01:45:25.967 INFO:teuthology.orchestra.run.smithi105.stdout: qatzip-libs x86_64 1.2.0-1.el9 appstream 47 k 2024-09-21T01:45:25.967 INFO:teuthology.orchestra.run.smithi105.stdout:Installing weak dependencies: 2024-09-21T01:45:25.967 INFO:teuthology.orchestra.run.smithi105.stdout: qatlib-service x86_64 24.02.0-1.el9 appstream 36 k 2024-09-21T01:45:25.967 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:45:25.967 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction Summary 2024-09-21T01:45:25.967 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T01:45:25.968 INFO:teuthology.orchestra.run.smithi105.stdout:Install 8 Packages 2024-09-21T01:45:25.968 INFO:teuthology.orchestra.run.smithi105.stdout:Upgrade 30 Packages 2024-09-21T01:45:25.968 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:45:25.968 INFO:teuthology.orchestra.run.smithi105.stdout:Total download size: 122 M 2024-09-21T01:45:25.968 INFO:teuthology.orchestra.run.smithi105.stdout:Downloading Packages: 2024-09-21T01:45:26.541 INFO:teuthology.orchestra.run.smithi105.stdout:(1/38): python3-packaging-20.9-5.el9.noarch.rpm 513 kB/s | 77 kB 00:00 2024-09-21T01:45:26.584 INFO:teuthology.orchestra.run.smithi105.stdout:(2/38): libnbd-1.20.2-2.el9.x86_64.rpm 885 kB/s | 171 kB 00:00 2024-09-21T01:45:26.617 INFO:teuthology.orchestra.run.smithi105.stdout:(3/38): lua-5.4.4-4.el9.x86_64.rpm 833 kB/s | 188 kB 00:00 2024-09-21T01:45:26.643 INFO:teuthology.orchestra.run.smithi105.stdout:(4/38): qatlib-service-24.02.0-1.el9.x86_64.rpm 611 kB/s | 36 kB 00:00 2024-09-21T01:45:26.668 INFO:teuthology.orchestra.run.smithi105.stdout:(5/38): qatlib-24.02.0-1.el9.x86_64.rpm 1.7 MB/s | 221 kB 00:00 2024-09-21T01:45:26.693 INFO:teuthology.orchestra.run.smithi105.stdout:(6/38): qatzip-libs-1.2.0-1.el9.x86_64.rpm 625 kB/s | 47 kB 00:00 2024-09-21T01:45:26.760 INFO:teuthology.orchestra.run.smithi105.stdout:(7/38): luarocks-3.9.2-5.el9.noarch.rpm 1.6 MB/s | 151 kB 00:00 2024-09-21T01:45:26.794 INFO:teuthology.orchestra.run.smithi105.stdout:(8/38): lua-devel-5.4.4-4.el9.x86_64.rpm 148 kB/s | 22 kB 00:00 2024-09-21T01:45:26.861 INFO:teuthology.orchestra.run.smithi105.stdout:(9/38): ceph-19.3.0-5120.g18432218.el9.x86_64.r 39 kB/s | 6.6 kB 00:00 2024-09-21T01:45:27.061 INFO:teuthology.orchestra.run.smithi105.stdout:(10/38): ceph-fuse-19.3.0-5120.g18432218.el9.x8 4.3 MB/s | 873 kB 00:00 2024-09-21T01:45:27.136 INFO:teuthology.orchestra.run.smithi105.stdout:(11/38): ceph-immutable-object-cache-19.3.0-512 1.9 MB/s | 148 kB 00:00 2024-09-21T01:45:27.295 INFO:teuthology.orchestra.run.smithi105.stdout:(12/38): ceph-base-19.3.0-5120.g18432218.el9.x8 10 MB/s | 5.5 MB 00:00 2024-09-21T01:45:27.345 INFO:teuthology.orchestra.run.smithi105.stdout:(13/38): ceph-mds-19.3.0-5120.g18432218.el9.x86 11 MB/s | 2.3 MB 00:00 2024-09-21T01:45:27.412 INFO:teuthology.orchestra.run.smithi105.stdout:(14/38): ceph-mgr-19.3.0-5120.g18432218.el9.x86 13 MB/s | 1.5 MB 00:00 2024-09-21T01:45:27.946 INFO:teuthology.orchestra.run.smithi105.stdout:(15/38): ceph-common-19.3.0-5120.g18432218.el9. 17 MB/s | 20 MB 00:01 2024-09-21T01:45:28.013 INFO:teuthology.orchestra.run.smithi105.stdout:(16/38): ceph-mon-19.3.0-5120.g18432218.el9.x86 7.4 MB/s | 5.0 MB 00:00 2024-09-21T01:45:28.055 INFO:teuthology.orchestra.run.smithi105.stdout:(17/38): ceph-selinux-19.3.0-5120.g18432218.el9 606 kB/s | 25 kB 00:00 2024-09-21T01:45:28.397 INFO:teuthology.orchestra.run.smithi105.stdout:(18/38): ceph-osd-19.3.0-5120.g18432218.el9.x86 18 MB/s | 18 MB 00:00 2024-09-21T01:45:28.439 INFO:teuthology.orchestra.run.smithi105.stdout:(19/38): libcephfs-devel-19.3.0-5120.g18432218. 767 kB/s | 32 kB 00:00 2024-09-21T01:45:28.578 INFO:teuthology.orchestra.run.smithi105.stdout:(20/38): libcephfs2-19.3.0-5120.g18432218.el9.x 5.4 MB/s | 766 kB 00:00 2024-09-21T01:45:28.715 INFO:teuthology.orchestra.run.smithi105.stdout:(21/38): ceph-radosgw-19.3.0-5120.g18432218.el9 14 MB/s | 10 MB 00:00 2024-09-21T01:45:28.739 INFO:teuthology.orchestra.run.smithi105.stdout:(22/38): libcephsqlite-19.3.0-5120.g18432218.el 984 kB/s | 158 kB 00:00 2024-09-21T01:45:28.765 INFO:teuthology.orchestra.run.smithi105.stdout:(23/38): librados-devel-19.3.0-5120.g18432218.e 2.5 MB/s | 127 kB 00:00 2024-09-21T01:45:28.857 INFO:teuthology.orchestra.run.smithi105.stdout:(24/38): libradosstriper1-19.3.0-5120.g18432218 5.4 MB/s | 507 kB 00:00 2024-09-21T01:45:28.924 INFO:teuthology.orchestra.run.smithi105.stdout:(25/38): librados2-19.3.0-5120.g18432218.el9.x8 18 MB/s | 3.4 MB 00:00 2024-09-21T01:45:29.158 INFO:teuthology.orchestra.run.smithi105.stdout:(26/38): librbd1-19.3.0-5120.g18432218.el9.x86_ 11 MB/s | 3.2 MB 00:00 2024-09-21T01:45:29.233 INFO:teuthology.orchestra.run.smithi105.stdout:(27/38): librgw2-19.3.0-5120.g18432218.el9.x86_ 17 MB/s | 5.3 MB 00:00 2024-09-21T01:45:29.258 INFO:teuthology.orchestra.run.smithi105.stdout:(28/38): python3-ceph-argparse-19.3.0-5120.g184 450 kB/s | 45 kB 00:00 2024-09-21T01:45:29.283 INFO:teuthology.orchestra.run.smithi105.stdout:(29/38): python3-ceph-common-19.3.0-5120.g18432 2.9 MB/s | 148 kB 00:00 2024-09-21T01:45:29.309 INFO:teuthology.orchestra.run.smithi105.stdout:(30/38): python3-cephfs-19.3.0-5120.g18432218.e 3.1 MB/s | 161 kB 00:00 2024-09-21T01:45:29.351 INFO:teuthology.orchestra.run.smithi105.stdout:(31/38): python3-rados-19.3.0-5120.g18432218.el 4.7 MB/s | 321 kB 00:00 2024-09-21T01:45:29.376 INFO:teuthology.orchestra.run.smithi105.stdout:(32/38): python3-rbd-19.3.0-5120.g18432218.el9. 4.5 MB/s | 307 kB 00:00 2024-09-21T01:45:29.402 INFO:teuthology.orchestra.run.smithi105.stdout:(33/38): python3-rgw-19.3.0-5120.g18432218.el9. 1.9 MB/s | 99 kB 00:00 2024-09-21T01:45:29.427 INFO:teuthology.orchestra.run.smithi105.stdout:(34/38): rbd-fuse-19.3.0-5120.g18432218.el9.x86 1.7 MB/s | 86 kB 00:00 2024-09-21T01:45:29.519 INFO:teuthology.orchestra.run.smithi105.stdout:(35/38): rbd-mirror-19.3.0-5120.g18432218.el9.x 27 MB/s | 3.1 MB 00:00 2024-09-21T01:45:29.544 INFO:teuthology.orchestra.run.smithi105.stdout:(36/38): rbd-nbd-19.3.0-5120.g18432218.el9.x86_ 1.4 MB/s | 170 kB 00:00 2024-09-21T01:45:29.569 INFO:teuthology.orchestra.run.smithi105.stdout:(37/38): ceph-mgr-modules-core-19.3.0-5120.g184 5.7 MB/s | 292 kB 00:00 2024-09-21T01:45:30.245 INFO:teuthology.orchestra.run.smithi105.stdout:(38/38): ceph-test-19.3.0-5120.g18432218.el9.x8 18 MB/s | 40 MB 00:02 2024-09-21T01:45:30.249 INFO:teuthology.orchestra.run.smithi105.stdout:-------------------------------------------------------------------------------- 2024-09-21T01:45:30.250 INFO:teuthology.orchestra.run.smithi105.stdout:Total 29 MB/s | 122 MB 00:04 2024-09-21T01:45:30.333 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction check 2024-09-21T01:45:30.390 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction check succeeded. 2024-09-21T01:45:30.390 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction test 2024-09-21T01:45:31.295 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction test succeeded. 2024-09-21T01:45:31.295 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction 2024-09-21T01:45:34.074 INFO:teuthology.orchestra.run.smithi105.stdout: Preparing : 1/1 2024-09-21T01:45:34.179 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : librados2-2:19.3.0-5120.g18432218.el9.x86_64 1/68 2024-09-21T01:45:34.307 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librados2-2:19.3.0-5120.g18432218.el9.x86_64 1/68 2024-09-21T01:45:34.355 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : libcephfs2-2:19.3.0-5120.g18432218.el9.x86_64 2/68 2024-09-21T01:45:35.025 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libcephfs2-2:19.3.0-5120.g18432218.el9.x86_64 2/68 2024-09-21T01:45:35.090 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : librgw2-2:19.3.0-5120.g18432218.el9.x86_64 3/68 2024-09-21T01:45:35.174 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librgw2-2:19.3.0-5120.g18432218.el9.x86_64 3/68 2024-09-21T01:45:35.225 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : python3-rados-2:19.3.0-5120.g18432218.el9.x86_64 4/68 2024-09-21T01:45:35.552 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libnbd-1.20.2-2.el9.x86_64 5/68 2024-09-21T01:45:35.626 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : librbd1-2:19.3.0-5120.g18432218.el9.x86_64 6/68 2024-09-21T01:45:35.692 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librbd1-2:19.3.0-5120.g18432218.el9.x86_64 6/68 2024-09-21T01:45:35.727 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : libcephsqlite-2:19.3.0-5120.g18432218.el9.x86_64 7/68 2024-09-21T01:45:35.841 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libcephsqlite-2:19.3.0-5120.g18432218.el9.x86_64 7/68 2024-09-21T01:45:35.878 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : libradosstriper1-2:19.3.0-5120.g18432218.el9.x86_6 8/68 2024-09-21T01:45:35.941 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libradosstriper1-2:19.3.0-5120.g18432218.el9.x86_6 8/68 2024-09-21T01:45:35.971 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : qatlib-service-24.02.0-1.el9.x86_64 9/68 2024-09-21T01:45:35.988 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: qatlib-service-24.02.0-1.el9.x86_64 9/68 2024-09-21T01:45:36.172 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: qatlib-24.02.0-1.el9.x86_64 10/68 2024-09-21T01:45:36.206 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : qatlib-24.02.0-1.el9.x86_64 10/68 2024-09-21T01:45:36.241 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : python3-ceph-argparse-2:19.3.0-5120.g18432218.el9. 11/68 2024-09-21T01:45:36.269 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : lua-5.4.4-4.el9.x86_64 12/68 2024-09-21T01:45:36.347 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 13/68 2024-09-21T01:45:36.401 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : luarocks-3.9.2-5.el9.noarch 14/68 2024-09-21T01:45:36.439 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : python3-cephfs-2:19.3.0-5120.g18432218.el9.x86_64 15/68 2024-09-21T01:45:36.500 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : qatzip-libs-1.2.0-1.el9.x86_64 16/68 2024-09-21T01:45:36.556 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : python3-rbd-2:19.3.0-5120.g18432218.el9.x86_64 17/68 2024-09-21T01:45:36.600 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : python3-rgw-2:19.3.0-5120.g18432218.el9.x86_64 18/68 2024-09-21T01:45:36.709 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : librados-devel-2:19.3.0-5120.g18432218.el9.x86_64 19/68 2024-09-21T01:45:36.764 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : python3-ceph-common-2:19.3.0-5120.g18432218.el9.x8 20/68 2024-09-21T01:45:39.315 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-common-2:19.3.0-5120.g18432218.el9.x86_64 21/68 2024-09-21T01:45:39.541 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : ceph-common-2:19.3.0-5120.g18432218.el9.x86_64 21/68 2024-09-21T01:45:40.247 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-common-2:19.3.0-5120.g18432218.el9.x86_64 21/68 2024-09-21T01:45:40.420 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : ceph-base-2:19.3.0-5120.g18432218.el9.x86_64 22/68 2024-09-21T01:45:40.492 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-base-2:19.3.0-5120.g18432218.el9.x86_64 22/68 2024-09-21T01:45:40.529 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : ceph-selinux-2:19.3.0-5120.g18432218.el9.x86_64 23/68 2024-09-21T01:45:49.108 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-selinux-2:19.3.0-5120.g18432218.el9.x86_64 23/68 2024-09-21T01:45:49.211 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : ceph-mds-2:19.3.0-5120.g18432218.el9.x86_64 24/68 2024-09-21T01:45:49.792 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mds-2:19.3.0-5120.g18432218.el9.x86_64 24/68 2024-09-21T01:45:49.862 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : ceph-mon-2:19.3.0-5120.g18432218.el9.x86_64 25/68 2024-09-21T01:45:51.830 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mon-2:19.3.0-5120.g18432218.el9.x86_64 25/68 2024-09-21T01:45:51.890 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : ceph-osd-2:19.3.0-5120.g18432218.el9.x86_64 26/68 2024-09-21T01:45:51.925 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-osd-2:19.3.0-5120.g18432218.el9.x86_64 26/68 2024-09-21T01:45:52.090 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-packaging-20.9-5.el9.noarch 27/68 2024-09-21T01:45:52.285 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : ceph-mgr-modules-core-2:19.3.0-5120.g18432218.el9. 28/68 2024-09-21T01:45:52.368 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : ceph-mgr-2:19.3.0-5120.g18432218.el9.x86_64 29/68 2024-09-21T01:45:52.374 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mgr-2:19.3.0-5120.g18432218.el9.x86_64 29/68 2024-09-21T01:45:53.618 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : ceph-2:19.3.0-5120.g18432218.el9.x86_64 30/68 2024-09-21T01:45:53.715 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : ceph-radosgw-2:19.3.0-5120.g18432218.el9.x86_64 31/68 2024-09-21T01:45:53.737 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-radosgw-2:19.3.0-5120.g18432218.el9.x86_64 31/68 2024-09-21T01:45:53.779 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : ceph-immutable-object-cache-2:19.3.0-5120.g1843221 32/68 2024-09-21T01:45:54.148 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.3.0-5120.g1843221 32/68 2024-09-21T01:45:54.221 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : rbd-mirror-2:19.3.0-5120.g18432218.el9.x86_64 33/68 2024-09-21T01:45:59.041 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: rbd-mirror-2:19.3.0-5120.g18432218.el9.x86_64 33/68 2024-09-21T01:45:59.407 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : ceph-test-2:19.3.0-5120.g18432218.el9.x86_64 34/68 2024-09-21T01:45:59.456 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : libcephfs-devel-2:19.3.0-5120.g18432218.el9.x86_64 35/68 2024-09-21T01:45:59.519 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : rbd-fuse-2:19.3.0-5120.g18432218.el9.x86_64 36/68 2024-09-21T01:45:59.643 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : rbd-nbd-2:19.3.0-5120.g18432218.el9.x86_64 37/68 2024-09-21T01:45:59.721 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : ceph-fuse-2:19.3.0-5120.g18432218.el9.x86_64 38/68 2024-09-21T01:45:59.751 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 39/68 2024-09-21T01:45:59.791 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 40/68 2024-09-21T01:45:59.798 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 41/68 2024-09-21T01:45:59.798 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 41/68 2024-09-21T01:45:59.834 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 41/68 2024-09-21T01:45:59.859 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 42/68 2024-09-21T01:45:59.864 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 43/68 2024-09-21T01:45:59.864 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 43/68 2024-09-21T01:45:59.902 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 43/68 2024-09-21T01:45:59.906 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 44/68 2024-09-21T01:45:59.906 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 44/68 2024-09-21T01:45:59.943 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 44/68 2024-09-21T01:45:59.968 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 45/68 2024-09-21T01:45:59.973 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85 46/68 2024-09-21T01:45:59.973 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85 46/68 2024-09-21T01:46:00.010 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85 46/68 2024-09-21T01:46:00.035 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 47/68 2024-09-21T01:46:00.035 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 48/68 2024-09-21T01:46:00.120 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 48/68 2024-09-21T01:46:00.145 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 49/68 2024-09-21T01:46:00.150 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 50/68 2024-09-21T01:46:00.150 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 50/68 2024-09-21T01:46:00.179 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 50/68 2024-09-21T01:46:00.184 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 51/68 2024-09-21T01:46:00.184 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 51/68 2024-09-21T01:46:00.221 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 51/68 2024-09-21T01:46:00.226 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 52/68 2024-09-21T01:46:00.226 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 52/68 2024-09-21T01:46:00.271 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 52/68 2024-09-21T01:46:00.272 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 53/68 2024-09-21T01:46:00.305 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 53/68 2024-09-21T01:46:00.312 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 54/68 2024-09-21T01:46:00.312 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 54/68 2024-09-21T01:46:00.426 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 54/68 2024-09-21T01:46:00.426 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 55/68 2024-09-21T01:46:00.459 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 55/68 2024-09-21T01:46:00.459 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 56/68 2024-09-21T01:46:00.543 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 56/68 2024-09-21T01:46:00.568 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 57/68 2024-09-21T01:46:00.594 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 58/68 2024-09-21T01:46:00.594 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 59/68 2024-09-21T01:46:00.687 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 59/68 2024-09-21T01:46:00.712 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 60/68 2024-09-21T01:46:00.712 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 61/68 2024-09-21T01:46:00.797 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 61/68 2024-09-21T01:46:00.821 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 62/68 2024-09-21T01:46:00.821 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 63/68 2024-09-21T01:46:00.906 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 63/68 2024-09-21T01:46:00.931 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 64/68 2024-09-21T01:46:00.956 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : python3-ceph-argparse-2:18.2.4-871.g6a1bdf85.el9.x 65/68 2024-09-21T01:46:00.981 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : python3-ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86 66/68 2024-09-21T01:46:01.015 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : ceph-mgr-modules-core-2:18.2.4-871.g6a1bdf85.el9.n 67/68 2024-09-21T01:46:01.015 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 68/68 2024-09-21T01:46:02.536 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 68/68 2024-09-21T01:46:02.536 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libnbd-1.20.2-2.el9.x86_64 1/68 2024-09-21T01:46:02.536 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : lua-5.4.4-4.el9.x86_64 2/68 2024-09-21T01:46:02.536 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 3/68 2024-09-21T01:46:02.536 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : qatlib-24.02.0-1.el9.x86_64 4/68 2024-09-21T01:46:02.536 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : qatlib-service-24.02.0-1.el9.x86_64 5/68 2024-09-21T01:46:02.537 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : qatzip-libs-1.2.0-1.el9.x86_64 6/68 2024-09-21T01:46:02.537 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 7/68 2024-09-21T01:46:02.537 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 8/68 2024-09-21T01:46:02.537 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-2:19.3.0-5120.g18432218.el9.x86_64 9/68 2024-09-21T01:46:02.537 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-2:18.2.4-871.g6a1bdf85.el9.x86_64 10/68 2024-09-21T01:46:02.537 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-base-2:19.3.0-5120.g18432218.el9.x86_64 11/68 2024-09-21T01:46:02.537 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-base-2:18.2.4-871.g6a1bdf85.el9.x86_64 12/68 2024-09-21T01:46:02.537 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-common-2:19.3.0-5120.g18432218.el9.x86_64 13/68 2024-09-21T01:46:02.537 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86_64 14/68 2024-09-21T01:46:02.537 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-fuse-2:19.3.0-5120.g18432218.el9.x86_64 15/68 2024-09-21T01:46:02.537 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 16/68 2024-09-21T01:46:02.538 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-immutable-object-cache-2:19.3.0-5120.g1843221 17/68 2024-09-21T01:46:02.538 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-immutable-object-cache-2:18.2.4-871.g6a1bdf85 18/68 2024-09-21T01:46:02.538 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mds-2:19.3.0-5120.g18432218.el9.x86_64 19/68 2024-09-21T01:46:02.538 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mds-2:18.2.4-871.g6a1bdf85.el9.x86_64 20/68 2024-09-21T01:46:02.538 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mgr-2:19.3.0-5120.g18432218.el9.x86_64 21/68 2024-09-21T01:46:02.538 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mgr-2:18.2.4-871.g6a1bdf85.el9.x86_64 22/68 2024-09-21T01:46:02.538 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mon-2:19.3.0-5120.g18432218.el9.x86_64 23/68 2024-09-21T01:46:02.538 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mon-2:18.2.4-871.g6a1bdf85.el9.x86_64 24/68 2024-09-21T01:46:02.538 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-osd-2:19.3.0-5120.g18432218.el9.x86_64 25/68 2024-09-21T01:46:02.538 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-osd-2:18.2.4-871.g6a1bdf85.el9.x86_64 26/68 2024-09-21T01:46:02.538 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-radosgw-2:19.3.0-5120.g18432218.el9.x86_64 27/68 2024-09-21T01:46:02.539 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-radosgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 28/68 2024-09-21T01:46:02.539 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-selinux-2:19.3.0-5120.g18432218.el9.x86_64 29/68 2024-09-21T01:46:02.539 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-selinux-2:18.2.4-871.g6a1bdf85.el9.x86_64 30/68 2024-09-21T01:46:02.539 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-test-2:19.3.0-5120.g18432218.el9.x86_64 31/68 2024-09-21T01:46:02.539 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-test-2:18.2.4-871.g6a1bdf85.el9.x86_64 32/68 2024-09-21T01:46:02.539 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libcephfs-devel-2:19.3.0-5120.g18432218.el9.x86_64 33/68 2024-09-21T01:46:02.539 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libcephfs-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 34/68 2024-09-21T01:46:02.539 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libcephfs2-2:19.3.0-5120.g18432218.el9.x86_64 35/68 2024-09-21T01:46:02.539 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libcephfs2-2:18.2.4-871.g6a1bdf85.el9.x86_64 36/68 2024-09-21T01:46:02.539 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libcephsqlite-2:19.3.0-5120.g18432218.el9.x86_64 37/68 2024-09-21T01:46:02.539 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libcephsqlite-2:18.2.4-871.g6a1bdf85.el9.x86_64 38/68 2024-09-21T01:46:02.539 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librados-devel-2:19.3.0-5120.g18432218.el9.x86_64 39/68 2024-09-21T01:46:02.540 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librados-devel-2:18.2.4-871.g6a1bdf85.el9.x86_64 40/68 2024-09-21T01:46:02.540 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librados2-2:19.3.0-5120.g18432218.el9.x86_64 41/68 2024-09-21T01:46:02.540 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librados2-2:18.2.4-871.g6a1bdf85.el9.x86_64 42/68 2024-09-21T01:46:02.540 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libradosstriper1-2:19.3.0-5120.g18432218.el9.x86_6 43/68 2024-09-21T01:46:02.540 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libradosstriper1-2:18.2.4-871.g6a1bdf85.el9.x86_64 44/68 2024-09-21T01:46:02.540 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librbd1-2:19.3.0-5120.g18432218.el9.x86_64 45/68 2024-09-21T01:46:02.540 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librbd1-2:18.2.4-871.g6a1bdf85.el9.x86_64 46/68 2024-09-21T01:46:02.540 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librgw2-2:19.3.0-5120.g18432218.el9.x86_64 47/68 2024-09-21T01:46:02.540 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librgw2-2:18.2.4-871.g6a1bdf85.el9.x86_64 48/68 2024-09-21T01:46:02.540 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-ceph-argparse-2:19.3.0-5120.g18432218.el9. 49/68 2024-09-21T01:46:02.540 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-ceph-argparse-2:18.2.4-871.g6a1bdf85.el9.x 50/68 2024-09-21T01:46:02.541 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-ceph-common-2:19.3.0-5120.g18432218.el9.x8 51/68 2024-09-21T01:46:02.541 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-ceph-common-2:18.2.4-871.g6a1bdf85.el9.x86 52/68 2024-09-21T01:46:02.541 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-cephfs-2:19.3.0-5120.g18432218.el9.x86_64 53/68 2024-09-21T01:46:02.541 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-cephfs-2:18.2.4-871.g6a1bdf85.el9.x86_64 54/68 2024-09-21T01:46:02.541 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-rados-2:19.3.0-5120.g18432218.el9.x86_64 55/68 2024-09-21T01:46:02.541 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-rados-2:18.2.4-871.g6a1bdf85.el9.x86_64 56/68 2024-09-21T01:46:02.541 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-rbd-2:19.3.0-5120.g18432218.el9.x86_64 57/68 2024-09-21T01:46:02.541 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-rbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 58/68 2024-09-21T01:46:02.541 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-rgw-2:19.3.0-5120.g18432218.el9.x86_64 59/68 2024-09-21T01:46:02.541 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-rgw-2:18.2.4-871.g6a1bdf85.el9.x86_64 60/68 2024-09-21T01:46:02.541 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : rbd-fuse-2:19.3.0-5120.g18432218.el9.x86_64 61/68 2024-09-21T01:46:02.541 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : rbd-fuse-2:18.2.4-871.g6a1bdf85.el9.x86_64 62/68 2024-09-21T01:46:02.542 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : rbd-mirror-2:19.3.0-5120.g18432218.el9.x86_64 63/68 2024-09-21T01:46:02.542 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : rbd-mirror-2:18.2.4-871.g6a1bdf85.el9.x86_64 64/68 2024-09-21T01:46:02.542 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : rbd-nbd-2:19.3.0-5120.g18432218.el9.x86_64 65/68 2024-09-21T01:46:02.542 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : rbd-nbd-2:18.2.4-871.g6a1bdf85.el9.x86_64 66/68 2024-09-21T01:46:02.542 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mgr-modules-core-2:19.3.0-5120.g18432218.el9. 67/68 2024-09-21T01:46:03.742 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mgr-modules-core-2:18.2.4-871.g6a1bdf85.el9.n 68/68 2024-09-21T01:46:03.742 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:03.742 INFO:teuthology.orchestra.run.smithi105.stdout:Upgraded: 2024-09-21T01:46:03.742 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.742 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-base-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.742 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-common-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.742 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-fuse-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.742 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-immutable-object-cache-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.742 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mds-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.743 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.743 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-modules-core-2:19.3.0-5120.g18432218.el9.noarch 2024-09-21T01:46:03.743 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mon-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.743 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-osd-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.743 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-radosgw-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.743 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-selinux-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.743 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-test-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.743 INFO:teuthology.orchestra.run.smithi105.stdout: libcephfs-devel-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.743 INFO:teuthology.orchestra.run.smithi105.stdout: libcephfs2-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.743 INFO:teuthology.orchestra.run.smithi105.stdout: libcephsqlite-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.743 INFO:teuthology.orchestra.run.smithi105.stdout: librados-devel-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.743 INFO:teuthology.orchestra.run.smithi105.stdout: librados2-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.743 INFO:teuthology.orchestra.run.smithi105.stdout: libradosstriper1-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.743 INFO:teuthology.orchestra.run.smithi105.stdout: librbd1-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.744 INFO:teuthology.orchestra.run.smithi105.stdout: librgw2-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.744 INFO:teuthology.orchestra.run.smithi105.stdout: python3-ceph-argparse-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.744 INFO:teuthology.orchestra.run.smithi105.stdout: python3-ceph-common-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.744 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cephfs-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.744 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rados-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.744 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rbd-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.744 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rgw-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.744 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-fuse-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.744 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-mirror-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.744 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-nbd-2:19.3.0-5120.g18432218.el9.x86_64 2024-09-21T01:46:03.744 INFO:teuthology.orchestra.run.smithi105.stdout:Installed: 2024-09-21T01:46:03.744 INFO:teuthology.orchestra.run.smithi105.stdout: libnbd-1.20.2-2.el9.x86_64 lua-5.4.4-4.el9.x86_64 2024-09-21T01:46:03.744 INFO:teuthology.orchestra.run.smithi105.stdout: lua-devel-5.4.4-4.el9.x86_64 luarocks-3.9.2-5.el9.noarch 2024-09-21T01:46:03.744 INFO:teuthology.orchestra.run.smithi105.stdout: python3-packaging-20.9-5.el9.noarch qatlib-24.02.0-1.el9.x86_64 2024-09-21T01:46:03.745 INFO:teuthology.orchestra.run.smithi105.stdout: qatlib-service-24.02.0-1.el9.x86_64 qatzip-libs-1.2.0-1.el9.x86_64 2024-09-21T01:46:03.745 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:03.745 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T01:46:04.195 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=1843221895adccfb3b692060251dd6a0183bc410 2024-09-21T01:46:04.344 DEBUG:teuthology.orchestra.run.smithi105:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-21T01:46:04.367 INFO:teuthology.orchestra.run.smithi105.stdout:19.3.0-5120.g18432218.el9 2024-09-21T01:46:04.367 INFO:teuthology.packaging:The installed version of ceph is 19.3.0-5120.g18432218.el9 2024-09-21T01:46:04.367 INFO:teuthology.task.install:The correct ceph version 19.3.0-5120.g18432218 is installed. 2024-09-21T01:46:04.368 INFO:teuthology.run_tasks:Running task print... 2024-09-21T01:46:04.383 INFO:teuthology.task.print:**** done install.upgrade 2024-09-21T01:46:04.383 INFO:teuthology.run_tasks:Running task ceph.restart... 2024-09-21T01:46:04.515 DEBUG:tasks.ceph.mon.a:waiting for process to exit 2024-09-21T01:46:04.515 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T01:46:04.526 INFO:tasks.ceph.mon.a:Stopped 2024-09-21T01:46:04.526 INFO:tasks.ceph.mon.a:Restarting daemon 2024-09-21T01:46:04.526 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i a 2024-09-21T01:46:04.569 INFO:tasks.ceph.mon.a:Started 2024-09-21T01:46:04.569 DEBUG:tasks.ceph.mon.b:waiting for process to exit 2024-09-21T01:46:04.569 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T01:46:04.581 INFO:tasks.ceph.mon.b:Stopped 2024-09-21T01:46:04.581 INFO:tasks.ceph.mon.b:Restarting daemon 2024-09-21T01:46:04.581 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i b 2024-09-21T01:46:04.624 INFO:tasks.ceph.mon.b:Started 2024-09-21T01:46:04.625 DEBUG:tasks.ceph.mon.c:waiting for process to exit 2024-09-21T01:46:04.625 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T01:46:04.637 INFO:tasks.ceph.mon.c:Stopped 2024-09-21T01:46:04.637 INFO:tasks.ceph.mon.c:Restarting daemon 2024-09-21T01:46:04.637 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster ceph -i c 2024-09-21T01:46:04.679 INFO:tasks.ceph.mon.c:Started 2024-09-21T01:46:04.680 DEBUG:tasks.ceph.mgr.x:waiting for process to exit 2024-09-21T01:46:04.680 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T01:46:04.709 INFO:tasks.ceph.mgr.x:Stopped 2024-09-21T01:46:04.709 INFO:tasks.ceph.mgr.x:Restarting daemon 2024-09-21T01:46:04.709 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i x 2024-09-21T01:46:04.752 INFO:tasks.ceph.mgr.x:Started 2024-09-21T01:46:04.752 DEBUG:tasks.ceph.mgr.y:waiting for process to exit 2024-09-21T01:46:04.752 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T01:46:04.777 INFO:tasks.ceph.mgr.y:Stopped 2024-09-21T01:46:04.778 INFO:tasks.ceph.mgr.y:Restarting daemon 2024-09-21T01:46:04.778 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster ceph -i y 2024-09-21T01:46:04.821 INFO:tasks.ceph.mgr.y:Started 2024-09-21T01:46:04.821 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-21T01:46:04.830 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T01:46:04.830 INFO:tasks.ceph.ceph_manager.ceph:waiting for mgr available 2024-09-21T01:46:04.831 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mgr dump --format=json 2024-09-21T01:46:09.922 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.919+0000 7ff33a726640 1 -- 172.21.15.105:0/181906694 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff3340966d0 msgr2=0x7ff314004800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:09.922 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.919+0000 7ff33a726640 1 --2- 172.21.15.105:0/181906694 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff3340966d0 0x7ff314004800 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7ff31c011c60 tx=0x7ff314005830 comp rx=0 tx=0).stop 2024-09-21T01:46:09.922 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.919+0000 7ff338cc3640 1 -- 172.21.15.105:0/181906694 reap_dead start 2024-09-21T01:46:09.922 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.920+0000 7ff33a726640 1 -- 172.21.15.105:0/181906694 shutdown_connections 2024-09-21T01:46:09.922 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.920+0000 7ff33a726640 1 -- 172.21.15.105:0/181906694 >> 172.21.15.105:0/181906694 conn(0x7ff33409e2a0 msgr2=0x7ff33409e6b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:09.922 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.920+0000 7ff33a726640 1 -- 172.21.15.105:0/181906694 shutdown_connections 2024-09-21T01:46:09.922 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.920+0000 7ff33a726640 1 -- 172.21.15.105:0/181906694 wait complete. 2024-09-21T01:46:09.923 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.921+0000 7ff33a726640 1 Processor -- start 2024-09-21T01:46:09.923 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.921+0000 7ff33a726640 1 -- start start 2024-09-21T01:46:09.925 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.922+0000 7ff33a726640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3340966d0 0x7ff334092910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:09.925 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.922+0000 7ff33a726640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff334123180 0x7ff334092e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:09.925 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.922+0000 7ff33a726640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff334093390 0x7ff3341cbbb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:09.925 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.922+0000 7ff33a726640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7ff334094e50 con 0x7ff3340966d0 2024-09-21T01:46:09.925 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.922+0000 7ff33a726640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7ff334094cd0 con 0x7ff334123180 2024-09-21T01:46:09.925 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.922+0000 7ff33a726640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7ff33412d890 con 0x7ff334093390 2024-09-21T01:46:09.925 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.922+0000 7ff333fff640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3340966d0 0x7ff334092910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:09.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.922+0000 7ff333fff640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3340966d0 0x7ff334092910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:33330/0 (socket says 172.21.15.105:33330) 2024-09-21T01:46:09.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.922+0000 7ff333fff640 1 -- 172.21.15.105:0/3378528860 learned_addr learned my addr 172.21.15.105:0/3378528860 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:09.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.922+0000 7ff338cc3640 1 --2- 172.21.15.105:0/3378528860 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff334093390 0x7ff3341cbbb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:09.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.922+0000 7ff333fff640 1 -- 172.21.15.105:0/3378528860 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff334093390 msgr2=0x7ff3341cbbb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:09.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.923+0000 7ff333fff640 1 --2- 172.21.15.105:0/3378528860 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff334093390 0x7ff3341cbbb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:09.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.923+0000 7ff333fff640 1 -- 172.21.15.105:0/3378528860 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff334123180 msgr2=0x7ff334092e50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:09.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.923+0000 7ff333fff640 1 --2- 172.21.15.105:0/3378528860 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff334123180 0x7ff334092e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:09.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.923+0000 7ff333fff640 1 -- 172.21.15.105:0/3378528860 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff334095ff0 con 0x7ff3340966d0 2024-09-21T01:46:09.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.923+0000 7ff333fff640 1 --2- 172.21.15.105:0/3378528860 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3340966d0 0x7ff334092910 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7ff324012aa0 tx=0x7ff324012f60 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:09.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.923+0000 7ff3317fa640 1 -- 172.21.15.105:0/3378528860 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff324022070 con 0x7ff3340966d0 2024-09-21T01:46:09.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.923+0000 7ff3317fa640 1 -- 172.21.15.105:0/3378528860 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7ff32401d480 con 0x7ff3340966d0 2024-09-21T01:46:09.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.923+0000 7ff33a726640 1 -- 172.21.15.105:0/3378528860 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff3341cc0f0 con 0x7ff3340966d0 2024-09-21T01:46:09.927 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.923+0000 7ff3317fa640 1 -- 172.21.15.105:0/3378528860 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff32401d8d0 con 0x7ff3340966d0 2024-09-21T01:46:09.927 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.923+0000 7ff33a726640 1 -- 172.21.15.105:0/3378528860 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff3341cc4a0 con 0x7ff3340966d0 2024-09-21T01:46:09.927 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.924+0000 7ff3317fa640 1 -- 172.21.15.105:0/3378528860 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7ff324025070 con 0x7ff3340966d0 2024-09-21T01:46:09.928 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.925+0000 7ff33a726640 1 -- 172.21.15.105:0/3378528860 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff334094170 con 0x7ff3340966d0 2024-09-21T01:46:09.928 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.925+0000 7ff32bfff640 1 -- 172.21.15.105:0/3378528860 >> v1:172.21.15.105:6816/3933313069 conn(0x7ff320047cf0 legacy=0x7ff32004a200 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.105:6816/3933313069 2024-09-21T01:46:09.928 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.925+0000 7ff3317fa640 1 -- 172.21.15.105:0/3378528860 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7ff32405e230 con 0x7ff3340966d0 2024-09-21T01:46:09.931 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:09.930+0000 7ff3317fa640 1 -- 172.21.15.105:0/3378528860 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7ff32405d3c0 con 0x7ff3340966d0 2024-09-21T01:46:10.100 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.098+0000 7ff33a726640 1 -- 172.21.15.105:0/3378528860 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "mgr dump", "format": "json"} v 0) -- 0x7ff33406a790 con 0x7ff3340966d0 2024-09-21T01:46:10.101 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.100+0000 7ff3317fa640 1 -- 172.21.15.105:0/3378528860 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json"}]=0 v6) ==== 74+0+107825 (secure 0 0 0) 0x7ff32405cf00 con 0x7ff3340966d0 2024-09-21T01:46:10.103 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:10.104 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.102+0000 7ff33a726640 1 -- 172.21.15.105:0/3378528860 >> v1:172.21.15.105:6816/3933313069 conn(0x7ff320047cf0 legacy=0x7ff32004a200 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:46:10.104 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.102+0000 7ff33a726640 1 -- 172.21.15.105:0/3378528860 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3340966d0 msgr2=0x7ff334092910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.104 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.102+0000 7ff33a726640 1 --2- 172.21.15.105:0/3378528860 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3340966d0 0x7ff334092910 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7ff324012aa0 tx=0x7ff324012f60 comp rx=0 tx=0).stop 2024-09-21T01:46:10.104 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.102+0000 7ff33a726640 1 -- 172.21.15.105:0/3378528860 shutdown_connections 2024-09-21T01:46:10.104 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.102+0000 7ff33a726640 1 --2- 172.21.15.105:0/3378528860 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff334093390 0x7ff3341cbbb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.104 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.102+0000 7ff33a726640 1 --2- 172.21.15.105:0/3378528860 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff334123180 0x7ff334092e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.104 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.102+0000 7ff33a726640 1 --2- 172.21.15.105:0/3378528860 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3340966d0 0x7ff334092910 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.105 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.102+0000 7ff33a726640 1 -- 172.21.15.105:0/3378528860 >> 172.21.15.105:0/3378528860 conn(0x7ff33409e2a0 msgr2=0x7ff334072a50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:10.105 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.102+0000 7ff33a726640 1 -- 172.21.15.105:0/3378528860 shutdown_connections 2024-09-21T01:46:10.105 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.102+0000 7ff33a726640 1 -- 172.21.15.105:0/3378528860 wait complete. 2024-09-21T01:46:10.115 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":6,"flags":0,"active_gid":4112,"active_name":"y","active_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6816","nonce":3933313069}]},"active_addr":"172.21.15.105:6816/3933313069","active_change":"2024-09-21T01:44:40.342853+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":4106,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.105:0","nonce":3837276043}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.105:0","nonce":3561680891}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.105:0","nonce":2570087590}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.105:0","nonce":2810478014}]}]} 2024-09-21T01:46:10.117 INFO:tasks.ceph.ceph_manager.ceph:mgr available! 2024-09-21T01:46:10.117 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up 2024-09-21T01:46:10.118 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T01:46:10.228 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.227+0000 7f9827472640 1 -- 172.21.15.105:0/1794172531 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f982006e9e0 msgr2=0x7f982006edf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.228 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.227+0000 7f9827472640 1 --2- 172.21.15.105:0/1794172531 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f982006e9e0 0x7f982006edf0 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f981c009830 tx=0x7f981c02a480 comp rx=0 tx=0).stop 2024-09-21T01:46:10.228 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.227+0000 7f9827472640 1 -- 172.21.15.105:0/1794172531 shutdown_connections 2024-09-21T01:46:10.229 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.227+0000 7f9827472640 1 --2- 172.21.15.105:0/1794172531 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f9820130260 0x7f9820137db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.229 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.227+0000 7f9827472640 1 --2- 172.21.15.105:0/1794172531 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f982006e9e0 0x7f982006edf0 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.229 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.227+0000 7f9827472640 1 -- 172.21.15.105:0/1794172531 >> 172.21.15.105:0/1794172531 conn(0x7f982009f000 msgr2=0x7f982009b4d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:10.229 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.227+0000 7f9827472640 1 -- 172.21.15.105:0/1794172531 shutdown_connections 2024-09-21T01:46:10.229 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.227+0000 7f9827472640 1 -- 172.21.15.105:0/1794172531 wait complete. 2024-09-21T01:46:10.231 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.228+0000 7f9827472640 1 Processor -- start 2024-09-21T01:46:10.231 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.228+0000 7f9827472640 1 -- start start 2024-09-21T01:46:10.231 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.228+0000 7f9827472640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f982006e9e0 0x7f98201c5d20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:10.231 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.228+0000 7f9827472640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f982006f330 0x7f98201c6260 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:10.231 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.228+0000 7f9827472640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9820130260 0x7f98201c67a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:10.231 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.228+0000 7f9827472640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f98201388d0 con 0x7f982006e9e0 2024-09-21T01:46:10.231 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.228+0000 7f9827472640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f9820138750 con 0x7f9820130260 2024-09-21T01:46:10.232 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.228+0000 7f9827472640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f9820138a50 con 0x7f982006f330 2024-09-21T01:46:10.232 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.229+0000 7f982520e640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f982006e9e0 0x7f98201c5d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:10.232 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.229+0000 7f982520e640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f982006e9e0 0x7f98201c5d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:33348/0 (socket says 172.21.15.105:33348) 2024-09-21T01:46:10.232 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.229+0000 7f982520e640 1 -- 172.21.15.105:0/3969672054 learned_addr learned my addr 172.21.15.105:0/3969672054 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:10.232 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.229+0000 7f9824a0d640 1 --2- 172.21.15.105:0/3969672054 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f982006f330 0x7f98201c6260 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:10.232 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.229+0000 7f9825a0f640 1 --2- 172.21.15.105:0/3969672054 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9820130260 0x7f98201c67a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:10.232 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.229+0000 7f982520e640 1 -- 172.21.15.105:0/3969672054 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f982006f330 msgr2=0x7f98201c6260 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.232 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.229+0000 7f982520e640 1 --2- 172.21.15.105:0/3969672054 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f982006f330 0x7f98201c6260 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.232 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.229+0000 7f982520e640 1 -- 172.21.15.105:0/3969672054 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9820130260 msgr2=0x7f98201c67a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.232 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.229+0000 7f982520e640 1 --2- 172.21.15.105:0/3969672054 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9820130260 0x7f98201c67a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.232 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.229+0000 7f982520e640 1 -- 172.21.15.105:0/3969672054 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9820094b40 con 0x7f982006e9e0 2024-09-21T01:46:10.232 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.229+0000 7f9824a0d640 1 --2- 172.21.15.105:0/3969672054 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f982006f330 0x7f98201c6260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:10.232 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.229+0000 7f9825a0f640 1 --2- 172.21.15.105:0/3969672054 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9820130260 0x7f98201c67a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:10.232 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.229+0000 7f982520e640 1 --2- 172.21.15.105:0/3969672054 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f982006e9e0 0x7f98201c5d20 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f981c009950 tx=0x7f981c033c20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:10.233 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.229+0000 7f9815ffb640 1 -- 172.21.15.105:0/3969672054 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f981c031d60 con 0x7f982006e9e0 2024-09-21T01:46:10.233 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.229+0000 7f9815ffb640 1 -- 172.21.15.105:0/3969672054 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f981c033e10 con 0x7f982006e9e0 2024-09-21T01:46:10.233 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.229+0000 7f9827472640 1 -- 172.21.15.105:0/3969672054 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f98200959f0 con 0x7f982006e9e0 2024-09-21T01:46:10.233 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.229+0000 7f9815ffb640 1 -- 172.21.15.105:0/3969672054 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f981c0332b0 con 0x7f982006e9e0 2024-09-21T01:46:10.233 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.229+0000 7f9827472640 1 -- 172.21.15.105:0/3969672054 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f9820092aa0 con 0x7f982006e9e0 2024-09-21T01:46:10.233 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.230+0000 7f9827472640 1 -- 172.21.15.105:0/3969672054 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f982006edf0 con 0x7f982006e9e0 2024-09-21T01:46:10.233 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.231+0000 7f9815ffb640 1 -- 172.21.15.105:0/3969672054 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7f981c033450 con 0x7f982006e9e0 2024-09-21T01:46:10.233 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.231+0000 7f9824a0d640 1 -- 172.21.15.105:0/3969672054 >> v1:172.21.15.105:6816/3933313069 conn(0x7f98040475f0 legacy=0x7f9804049b00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.105:6816/3933313069 2024-09-21T01:46:10.234 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.231+0000 7f9815ffb640 1 -- 172.21.15.105:0/3969672054 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7f981c07df80 con 0x7f982006e9e0 2024-09-21T01:46:10.235 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.234+0000 7f9815ffb640 1 -- 172.21.15.105:0/3969672054 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7f981c03f030 con 0x7f982006e9e0 2024-09-21T01:46:10.372 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.371+0000 7f9827472640 1 -- 172.21.15.105:0/3969672054 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f982006a820 con 0x7f982006e9e0 2024-09-21T01:46:10.373 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.372+0000 7f9815ffb640 1 -- 172.21.15.105:0/3969672054 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v19) ==== 74+0+8855 (secure 0 0 0) 0x7f981c07d110 con 0x7f982006e9e0 2024-09-21T01:46:10.373 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:10.373 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":19,"fsid":"355399d3-6490-4ad0-ac52-d793648ff276","created":"2024-09-21T01:44:37.282024+0000","modified":"2024-09-21T01:45:03.546273+0000","last_up_change":"2024-09-21T01:44:48.482537+0000","last_in_change":"2024-09-21T01:44:39.666350+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"reef","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T01:44:50.380956+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T01:44:54.210449+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"03218bdb-ec48-44c7-bbf5-31cb53a3cea7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6800","nonce":4053719786}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6801","nonce":4053719786}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6803","nonce":4053719786}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6802","nonce":4053719786}]},"public_addr":"172.21.15.105:6800/4053719786","cluster_addr":"172.21.15.105:6801/4053719786","heartbeat_back_addr":"172.21.15.105:6803/4053719786","heartbeat_front_addr":"172.21.15.105:6802/4053719786","state":["exists","up"]},{"osd":1,"uuid":"5528ff44-922a-4e44-bb69-95fd9544d8dd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6812","nonce":1313742696}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6813","nonce":1313742696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6815","nonce":1313742696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6814","nonce":1313742696}]},"public_addr":"172.21.15.105:6812/1313742696","cluster_addr":"172.21.15.105:6813/1313742696","heartbeat_back_addr":"172.21.15.105:6815/1313742696","heartbeat_front_addr":"172.21.15.105:6814/1313742696","state":["exists","up"]},{"osd":2,"uuid":"df56ff35-71e9-4d46-8abb-84265eb6e1ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6804","nonce":2159605016}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6805","nonce":2159605016}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6807","nonce":2159605016}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6806","nonce":2159605016}]},"public_addr":"172.21.15.105:6804/2159605016","cluster_addr":"172.21.15.105:6805/2159605016","heartbeat_back_addr":"172.21.15.105:6807/2159605016","heartbeat_front_addr":"172.21.15.105:6806/2159605016","state":["exists","up"]},{"osd":3,"uuid":"012d9668-f1d3-4268-b0eb-15fa845ef17c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6808","nonce":2514115267}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6809","nonce":2514115267}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6811","nonce":2514115267}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6810","nonce":2514115267}]},"public_addr":"172.21.15.105:6808/2514115267","cluster_addr":"172.21.15.105:6809/2514115267","heartbeat_back_addr":"172.21.15.105:6811/2514115267","heartbeat_front_addr":"172.21.15.105:6810/2514115267","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.554795+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.869400+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.601687+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.822680+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T01:46:10.375 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.373+0000 7f9827472640 1 -- 172.21.15.105:0/3969672054 >> v1:172.21.15.105:6816/3933313069 conn(0x7f98040475f0 legacy=0x7f9804049b00 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:46:10.375 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.373+0000 7f9827472640 1 -- 172.21.15.105:0/3969672054 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f982006e9e0 msgr2=0x7f98201c5d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.375 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.373+0000 7f9827472640 1 --2- 172.21.15.105:0/3969672054 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f982006e9e0 0x7f98201c5d20 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f981c009950 tx=0x7f981c033c20 comp rx=0 tx=0).stop 2024-09-21T01:46:10.375 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.373+0000 7f9827472640 1 -- 172.21.15.105:0/3969672054 shutdown_connections 2024-09-21T01:46:10.375 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.373+0000 7f9827472640 1 --2- 172.21.15.105:0/3969672054 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9820130260 0x7f98201c67a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.375 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.373+0000 7f9827472640 1 --2- 172.21.15.105:0/3969672054 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f982006f330 0x7f98201c6260 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.375 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.373+0000 7f9827472640 1 --2- 172.21.15.105:0/3969672054 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f982006e9e0 0x7f98201c5d20 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.376 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.373+0000 7f9827472640 1 -- 172.21.15.105:0/3969672054 >> 172.21.15.105:0/3969672054 conn(0x7f982009f000 msgr2=0x7f9820132d80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:10.376 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.373+0000 7f9827472640 1 -- 172.21.15.105:0/3969672054 shutdown_connections 2024-09-21T01:46:10.376 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.373+0000 7f9827472640 1 -- 172.21.15.105:0/3969672054 wait complete. 2024-09-21T01:46:10.387 INFO:tasks.ceph.ceph_manager.ceph:all up! 2024-09-21T01:46:10.387 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T01:46:10.499 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.498+0000 7f2e53a68640 1 -- 172.21.15.105:0/1388192589 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2e4c12a400 msgr2=0x7f2e4c12a810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.500 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.498+0000 7f2e53a68640 1 --2- 172.21.15.105:0/1388192589 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2e4c12a400 0x7f2e4c12a810 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f2e3c009860 tx=0x7f2e3c02a460 comp rx=0 tx=0).stop 2024-09-21T01:46:10.500 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.498+0000 7f2e53a68640 1 -- 172.21.15.105:0/1388192589 shutdown_connections 2024-09-21T01:46:10.500 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.498+0000 7f2e53a68640 1 --2- 172.21.15.105:0/1388192589 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2e4c12a400 0x7f2e4c12a810 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.500 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.498+0000 7f2e53a68640 1 --2- 172.21.15.105:0/1388192589 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2e4c073960 0x7f2e4c129ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.500 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.498+0000 7f2e53a68640 1 -- 172.21.15.105:0/1388192589 >> 172.21.15.105:0/1388192589 conn(0x7f2e4c09fc30 msgr2=0x7f2e4c0a0040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:10.500 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.498+0000 7f2e53a68640 1 -- 172.21.15.105:0/1388192589 shutdown_connections 2024-09-21T01:46:10.500 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.498+0000 7f2e53a68640 1 -- 172.21.15.105:0/1388192589 wait complete. 2024-09-21T01:46:10.503 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.499+0000 7f2e53a68640 1 Processor -- start 2024-09-21T01:46:10.503 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.499+0000 7f2e53a68640 1 -- start start 2024-09-21T01:46:10.503 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.500+0000 7f2e53a68640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2e4c073960 0x7f2e4c133cc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:10.503 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.500+0000 7f2e53a68640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2e4c12a400 0x7f2e4c134200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:10.503 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.500+0000 7f2e53a68640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2e4c12d4a0 0x7f2e4c134740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:10.503 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.500+0000 7f2e53a68640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f2e4c1363e0 con 0x7f2e4c12a400 2024-09-21T01:46:10.503 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.500+0000 7f2e53a68640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f2e4c136260 con 0x7f2e4c073960 2024-09-21T01:46:10.503 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.500+0000 7f2e53a68640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f2e4c136560 con 0x7f2e4c12d4a0 2024-09-21T01:46:10.503 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.500+0000 7f2e51804640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2e4c073960 0x7f2e4c133cc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:10.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.500+0000 7f2e51804640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2e4c073960 0x7f2e4c133cc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:59842/0 (socket says 172.21.15.105:59842) 2024-09-21T01:46:10.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.500+0000 7f2e51804640 1 -- 172.21.15.105:0/409879655 learned_addr learned my addr 172.21.15.105:0/409879655 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:10.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.500+0000 7f2e51003640 1 --2- 172.21.15.105:0/409879655 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2e4c12a400 0x7f2e4c134200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:10.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.500+0000 7f2e52005640 1 --2- 172.21.15.105:0/409879655 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2e4c12d4a0 0x7f2e4c134740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:10.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.500+0000 7f2e51804640 1 -- 172.21.15.105:0/409879655 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2e4c12d4a0 msgr2=0x7f2e4c134740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.500+0000 7f2e51804640 1 --2- 172.21.15.105:0/409879655 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2e4c12d4a0 0x7f2e4c134740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.500+0000 7f2e51804640 1 -- 172.21.15.105:0/409879655 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2e4c12a400 msgr2=0x7f2e4c134200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.500+0000 7f2e51804640 1 --2- 172.21.15.105:0/409879655 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2e4c12a400 0x7f2e4c134200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.500+0000 7f2e51804640 1 -- 172.21.15.105:0/409879655 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2e4c1336f0 con 0x7f2e4c073960 2024-09-21T01:46:10.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.500+0000 7f2e52005640 1 --2- 172.21.15.105:0/409879655 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2e4c12d4a0 0x7f2e4c134740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:10.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.500+0000 7f2e51003640 1 --2- 172.21.15.105:0/409879655 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2e4c12a400 0x7f2e4c134200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:10.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.500+0000 7f2e51804640 1 --2- 172.21.15.105:0/409879655 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2e4c073960 0x7f2e4c133cc0 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f2e4800ea20 tx=0x7f2e4800eee0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:10.505 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.501+0000 7f2e427fc640 1 -- 172.21.15.105:0/409879655 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e4800ce40 con 0x7f2e4c073960 2024-09-21T01:46:10.505 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.501+0000 7f2e427fc640 1 -- 172.21.15.105:0/409879655 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2e48004020 con 0x7f2e4c073960 2024-09-21T01:46:10.505 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.501+0000 7f2e53a68640 1 -- 172.21.15.105:0/409879655 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2e4c1330a0 con 0x7f2e4c073960 2024-09-21T01:46:10.505 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.501+0000 7f2e427fc640 1 -- 172.21.15.105:0/409879655 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2e48004470 con 0x7f2e4c073960 2024-09-21T01:46:10.505 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.501+0000 7f2e53a68640 1 -- 172.21.15.105:0/409879655 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f2e4c1c3a10 con 0x7f2e4c073960 2024-09-21T01:46:10.505 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.502+0000 7f2e53a68640 1 -- 172.21.15.105:0/409879655 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2e4c132960 con 0x7f2e4c073960 2024-09-21T01:46:10.505 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.502+0000 7f2e427fc640 1 -- 172.21.15.105:0/409879655 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7f2e480035f0 con 0x7f2e4c073960 2024-09-21T01:46:10.506 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.502+0000 7f2e427fc640 1 -- 172.21.15.105:0/409879655 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7f2e4805d6e0 con 0x7f2e4c073960 2024-09-21T01:46:10.506 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.502+0000 7f2e51003640 1 -- 172.21.15.105:0/409879655 >> v1:172.21.15.105:6816/3933313069 conn(0x7f2e300475f0 legacy=0x7f2e30049b00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.105:6816/3933313069 2024-09-21T01:46:10.507 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.506+0000 7f2e427fc640 1 -- 172.21.15.105:0/409879655 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7f2e48009490 con 0x7f2e4c073960 2024-09-21T01:46:10.651 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.649+0000 7f2e53a68640 1 -- 172.21.15.105:0/409879655 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f2e4c129ec0 con 0x7f2e4c073960 2024-09-21T01:46:10.651 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.650+0000 7f2e427fc640 1 -- 172.21.15.105:0/409879655 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v19) ==== 74+0+8855 (secure 0 0 0) 0x7f2e4805c870 con 0x7f2e4c073960 2024-09-21T01:46:10.651 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:10.652 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":19,"fsid":"355399d3-6490-4ad0-ac52-d793648ff276","created":"2024-09-21T01:44:37.282024+0000","modified":"2024-09-21T01:45:03.546273+0000","last_up_change":"2024-09-21T01:44:48.482537+0000","last_in_change":"2024-09-21T01:44:39.666350+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"reef","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T01:44:50.380956+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T01:44:54.210449+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"03218bdb-ec48-44c7-bbf5-31cb53a3cea7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6800","nonce":4053719786}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6801","nonce":4053719786}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6803","nonce":4053719786}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6802","nonce":4053719786}]},"public_addr":"172.21.15.105:6800/4053719786","cluster_addr":"172.21.15.105:6801/4053719786","heartbeat_back_addr":"172.21.15.105:6803/4053719786","heartbeat_front_addr":"172.21.15.105:6802/4053719786","state":["exists","up"]},{"osd":1,"uuid":"5528ff44-922a-4e44-bb69-95fd9544d8dd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6812","nonce":1313742696}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6813","nonce":1313742696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6815","nonce":1313742696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6814","nonce":1313742696}]},"public_addr":"172.21.15.105:6812/1313742696","cluster_addr":"172.21.15.105:6813/1313742696","heartbeat_back_addr":"172.21.15.105:6815/1313742696","heartbeat_front_addr":"172.21.15.105:6814/1313742696","state":["exists","up"]},{"osd":2,"uuid":"df56ff35-71e9-4d46-8abb-84265eb6e1ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6804","nonce":2159605016}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6805","nonce":2159605016}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6807","nonce":2159605016}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6806","nonce":2159605016}]},"public_addr":"172.21.15.105:6804/2159605016","cluster_addr":"172.21.15.105:6805/2159605016","heartbeat_back_addr":"172.21.15.105:6807/2159605016","heartbeat_front_addr":"172.21.15.105:6806/2159605016","state":["exists","up"]},{"osd":3,"uuid":"012d9668-f1d3-4268-b0eb-15fa845ef17c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6808","nonce":2514115267}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6809","nonce":2514115267}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6811","nonce":2514115267}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6810","nonce":2514115267}]},"public_addr":"172.21.15.105:6808/2514115267","cluster_addr":"172.21.15.105:6809/2514115267","heartbeat_back_addr":"172.21.15.105:6811/2514115267","heartbeat_front_addr":"172.21.15.105:6810/2514115267","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.554795+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.869400+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.601687+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.822680+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T01:46:10.654 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.652+0000 7f2e53a68640 1 -- 172.21.15.105:0/409879655 >> v1:172.21.15.105:6816/3933313069 conn(0x7f2e300475f0 legacy=0x7f2e30049b00 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:46:10.654 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.652+0000 7f2e53a68640 1 -- 172.21.15.105:0/409879655 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2e4c073960 msgr2=0x7f2e4c133cc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.654 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.652+0000 7f2e53a68640 1 --2- 172.21.15.105:0/409879655 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2e4c073960 0x7f2e4c133cc0 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f2e4800ea20 tx=0x7f2e4800eee0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.654 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.652+0000 7f2e53a68640 1 -- 172.21.15.105:0/409879655 shutdown_connections 2024-09-21T01:46:10.654 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.652+0000 7f2e53a68640 1 --2- 172.21.15.105:0/409879655 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2e4c12d4a0 0x7f2e4c134740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.654 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.652+0000 7f2e53a68640 1 --2- 172.21.15.105:0/409879655 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2e4c12a400 0x7f2e4c134200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.654 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.652+0000 7f2e53a68640 1 --2- 172.21.15.105:0/409879655 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2e4c073960 0x7f2e4c133cc0 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.654 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.652+0000 7f2e53a68640 1 -- 172.21.15.105:0/409879655 >> 172.21.15.105:0/409879655 conn(0x7f2e4c09fc30 msgr2=0x7f2e4c099700 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:10.654 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.652+0000 7f2e53a68640 1 -- 172.21.15.105:0/409879655 shutdown_connections 2024-09-21T01:46:10.654 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.652+0000 7f2e53a68640 1 -- 172.21.15.105:0/409879655 wait complete. 2024-09-21T01:46:10.665 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 flush_pg_stats 2024-09-21T01:46:10.666 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 flush_pg_stats 2024-09-21T01:46:10.666 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 flush_pg_stats 2024-09-21T01:46:10.667 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 flush_pg_stats 2024-09-21T01:46:10.789 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.787+0000 7f21fac20640 1 -- 172.21.15.105:0/3541880558 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f21f412f5d0 msgr2=0x7f21f4134950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.789 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.787+0000 7f21fac20640 1 --2- 172.21.15.105:0/3541880558 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f21f412f5d0 0x7f21f4134950 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f21e0009830 tx=0x7f21e002a440 comp rx=0 tx=0).stop 2024-09-21T01:46:10.789 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.788+0000 7f21fac20640 1 -- 172.21.15.105:0/3541880558 shutdown_connections 2024-09-21T01:46:10.789 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.788+0000 7f21fac20640 1 --2- 172.21.15.105:0/3541880558 >> v2:172.21.15.105:3300/0 conn(0x7f21f4130210 0x7f21f4136ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.789 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.788+0000 7f21fac20640 1 --2- 172.21.15.105:0/3541880558 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f21f412f5d0 0x7f21f4134950 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.789 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.788+0000 7f21fac20640 1 -- 172.21.15.105:0/3541880558 >> 172.21.15.105:0/3541880558 conn(0x7f21f40a1e40 msgr2=0x7f21f40a2250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:10.790 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.788+0000 7f21fac20640 1 -- 172.21.15.105:0/3541880558 shutdown_connections 2024-09-21T01:46:10.790 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.788+0000 7f21fac20640 1 -- 172.21.15.105:0/3541880558 wait complete. 2024-09-21T01:46:10.790 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.788+0000 7f7b1e092640 1 -- 172.21.15.105:0/2889288056 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f7b18131eb0 msgr2=0x7f7b181342d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.790 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.788+0000 7f7b1e092640 1 --2- 172.21.15.105:0/2889288056 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f7b18131eb0 0x7f7b181342d0 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f7b10009be0 tx=0x7f7b1002a870 comp rx=0 tx=0).stop 2024-09-21T01:46:10.790 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.788+0000 7f7b1e092640 1 -- 172.21.15.105:0/2889288056 shutdown_connections 2024-09-21T01:46:10.791 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.788+0000 7f7b1e092640 1 --2- 172.21.15.105:0/2889288056 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f7b18131eb0 0x7f7b181342d0 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.791 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.788+0000 7f7b1e092640 1 --2- 172.21.15.105:0/2889288056 >> v2:172.21.15.105:3300/0 conn(0x7f7b18072380 0x7f7b18131970 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.791 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.788+0000 7f7b1e092640 1 -- 172.21.15.105:0/2889288056 >> 172.21.15.105:0/2889288056 conn(0x7f7b1809f000 msgr2=0x7f7b1809b4d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:10.791 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.788+0000 7f7b1e092640 1 -- 172.21.15.105:0/2889288056 shutdown_connections 2024-09-21T01:46:10.791 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.788+0000 7f7b1e092640 1 -- 172.21.15.105:0/2889288056 wait complete. 2024-09-21T01:46:10.791 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.789+0000 7f21fac20640 1 Processor -- start 2024-09-21T01:46:10.791 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.789+0000 7f7b1e092640 1 Processor -- start 2024-09-21T01:46:10.792 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.790+0000 7f21fac20640 1 -- start start 2024-09-21T01:46:10.792 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.790+0000 7f7b1e092640 1 -- start start 2024-09-21T01:46:10.792 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.791+0000 7f7b1e092640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f7b18071860 0x7f7b181c39e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:10.792 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.791+0000 7f21fac20640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f21f40716c0 0x7f21f409b4b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:10.793 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.791+0000 7f7b1e092640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7b18072380 0x7f7b181c3f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:10.793 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.791+0000 7f7b177fe640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f7b18071860 0x7f7b181c39e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:10.793 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.791+0000 7f7b177fe640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f7b18071860 0x7f7b181c39e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:59852/0 (socket says 172.21.15.105:59852) 2024-09-21T01:46:10.793 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.791+0000 7f21fac20640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f21f412f5d0 0x7f21f409b9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:10.794 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.791+0000 7f7b177fe640 1 -- 172.21.15.105:0/2497702377 learned_addr learned my addr 172.21.15.105:0/2497702377 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:10.794 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.791+0000 7f7b1e092640 1 --2- 172.21.15.105:0/2497702377 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f7b18131eb0 0x7f7b181c4460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:10.794 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.791+0000 7f21f89bc640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f21f40716c0 0x7f21f409b4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:10.794 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.791+0000 7f7b16ffd640 1 --2- 172.21.15.105:0/2497702377 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7b18072380 0x7f7b181c3f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:10.794 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.791+0000 7f21f89bc640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f21f40716c0 0x7f21f409b4b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:59860/0 (socket says 172.21.15.105:59860) 2024-09-21T01:46:10.795 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.791+0000 7f7b1e092640 1 -- 172.21.15.105:0/2497702377 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f7b1806f4e0 con 0x7f7b18072380 2024-09-21T01:46:10.795 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.791+0000 7f7b17fff640 1 --2- 172.21.15.105:0/2497702377 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f7b18131eb0 0x7f7b181c4460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:10.795 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f7b1e092640 1 -- 172.21.15.105:0/2497702377 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f7b1806f360 con 0x7f7b18071860 2024-09-21T01:46:10.795 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f7b1e092640 1 -- 172.21.15.105:0/2497702377 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f7b1806f660 con 0x7f7b18131eb0 2024-09-21T01:46:10.796 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f21f3fff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f21f412f5d0 0x7f21f409b9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:10.796 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f21f3fff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f21f412f5d0 0x7f21f409b9f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:53566/0 (socket says 172.21.15.105:53566) 2024-09-21T01:46:10.796 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f21f89bc640 1 -- 172.21.15.105:0/4291667580 learned_addr learned my addr 172.21.15.105:0/4291667580 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:10.796 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f21fac20640 1 --2- 172.21.15.105:0/4291667580 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f21f4130210 0x7f21f409bf30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:10.796 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f21fac20640 1 -- 172.21.15.105:0/4291667580 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f21f41385c0 con 0x7f21f4130210 2024-09-21T01:46:10.796 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f21fac20640 1 -- 172.21.15.105:0/4291667580 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f21f4138210 con 0x7f21f40716c0 2024-09-21T01:46:10.796 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f21fac20640 1 -- 172.21.15.105:0/4291667580 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f21f4138740 con 0x7f21f412f5d0 2024-09-21T01:46:10.796 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f21f91bd640 1 --2- 172.21.15.105:0/4291667580 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f21f4130210 0x7f21f409bf30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:10.797 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f21f89bc640 1 -- 172.21.15.105:0/4291667580 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f21f412f5d0 msgr2=0x7f21f409b9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.797 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f21f89bc640 1 --2- 172.21.15.105:0/4291667580 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f21f412f5d0 0x7f21f409b9f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.797 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f21f89bc640 1 -- 172.21.15.105:0/4291667580 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f21f4130210 msgr2=0x7f21f409bf30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.797 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f21f89bc640 1 --2- 172.21.15.105:0/4291667580 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f21f4130210 0x7f21f409bf30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.797 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f21f89bc640 1 -- 172.21.15.105:0/4291667580 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f21f4133850 con 0x7f21f40716c0 2024-09-21T01:46:10.797 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f7b17fff640 1 -- 172.21.15.105:0/2497702377 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f7b18071860 msgr2=0x7f7b181c39e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.797 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f7b17fff640 1 --2- 172.21.15.105:0/2497702377 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f7b18071860 0x7f7b181c39e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.797 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f7b17fff640 1 -- 172.21.15.105:0/2497702377 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7b18072380 msgr2=0x7f7b181c3f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.798 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f21f3fff640 1 --2- 172.21.15.105:0/4291667580 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f21f412f5d0 0x7f21f409b9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T01:46:10.798 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f21f89bc640 1 --2- 172.21.15.105:0/4291667580 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f21f40716c0 0x7f21f409b4b0 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f21e800c430 tx=0x7f21e800c8f0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:10.798 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f7b17fff640 1 --2- 172.21.15.105:0/2497702377 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7b18072380 0x7f7b181c3f20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.798 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f7b17fff640 1 -- 172.21.15.105:0/2497702377 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7b1812edb0 con 0x7f7b18131eb0 2024-09-21T01:46:10.798 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.792+0000 7f7b16ffd640 1 --2- 172.21.15.105:0/2497702377 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7b18072380 0x7f7b181c3f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:10.799 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.793+0000 7f21f17fa640 1 -- 172.21.15.105:0/4291667580 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f21e8011790 con 0x7f21f40716c0 2024-09-21T01:46:10.799 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.793+0000 7f21fac20640 1 -- 172.21.15.105:0/4291667580 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f21f41329a0 con 0x7f21f40716c0 2024-09-21T01:46:10.799 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.793+0000 7f21f17fa640 1 -- 172.21.15.105:0/4291667580 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f21e8017070 con 0x7f21f40716c0 2024-09-21T01:46:10.799 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.793+0000 7f21f17fa640 1 -- 172.21.15.105:0/4291667580 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f21e8011ca0 con 0x7f21f40716c0 2024-09-21T01:46:10.799 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.793+0000 7f21fac20640 1 -- 172.21.15.105:0/4291667580 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f21f4130970 con 0x7f21f40716c0 2024-09-21T01:46:10.800 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.793+0000 7f7b17fff640 1 --2- 172.21.15.105:0/2497702377 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f7b18131eb0 0x7f7b181c4460 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f7b18072eb0 tx=0x7f7b10033c70 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:10.800 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.794+0000 7f7afffff640 1 -- 172.21.15.105:0/2497702377 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b1003f020 con 0x7f7b18131eb0 2024-09-21T01:46:10.800 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.794+0000 7f7afffff640 1 -- 172.21.15.105:0/2497702377 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f7b10033070 con 0x7f7b18131eb0 2024-09-21T01:46:10.800 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.794+0000 7f7afffff640 1 -- 172.21.15.105:0/2497702377 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7b10036040 con 0x7f7b18131eb0 2024-09-21T01:46:10.800 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.794+0000 7f7b1e092640 1 -- 172.21.15.105:0/2497702377 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7b181262f0 con 0x7f7b18131eb0 2024-09-21T01:46:10.800 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.794+0000 7f21fac20640 1 -- 172.21.15.105:0/4291667580 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f21f412fb00 con 0x7f21f40716c0 2024-09-21T01:46:10.800 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.795+0000 7f21f17fa640 1 -- 172.21.15.105:0/4291667580 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7f21e801c460 con 0x7f21f40716c0 2024-09-21T01:46:10.801 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.795+0000 7f7b1e092640 1 -- 172.21.15.105:0/2497702377 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f7b18125ba0 con 0x7f7b18131eb0 2024-09-21T01:46:10.801 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.795+0000 7f21f17fa640 1 -- 172.21.15.105:0/4291667580 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7f21e805cb10 con 0x7f21f40716c0 2024-09-21T01:46:10.801 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.795+0000 7f21f17fa640 1 -- 172.21.15.105:0/4291667580 --> v1:172.21.15.105:6804/2159605016 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f21e8009e50 con 0x7f21d804dd30 2024-09-21T01:46:10.801 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.795+0000 7f21f17fa640 1 -- 172.21.15.105:0/4291667580 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_get_version_reply(handle=1 version=19) ==== 24+0+0 (secure 0 0 0) 0x7f21e803a2a0 con 0x7f21f40716c0 2024-09-21T01:46:10.801 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.795+0000 7f21f3fff640 1 -- 172.21.15.105:0/4291667580 >> v1:172.21.15.105:6816/3933313069 conn(0x7f21d80475f0 legacy=0x7f21d8049b00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.105:6816/3933313069 2024-09-21T01:46:10.802 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.796+0000 7f7afffff640 1 -- 172.21.15.105:0/2497702377 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7f7b10042020 con 0x7f7b18131eb0 2024-09-21T01:46:10.802 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.796+0000 7f7b1e092640 1 -- 172.21.15.105:0/2497702377 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f7b18133160 con 0x7f7b18131eb0 2024-09-21T01:46:10.802 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.796+0000 7f7b177fe640 1 -- 172.21.15.105:0/2497702377 >> v1:172.21.15.105:6816/3933313069 conn(0x7f7b0404ba50 legacy=0x7f7b0404df60 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.105:6816/3933313069 2024-09-21T01:46:10.802 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.796+0000 7f7afffff640 1 -- 172.21.15.105:0/2497702377 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7f7b1007d2a0 con 0x7f7b18131eb0 2024-09-21T01:46:10.802 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.796+0000 7f7afffff640 1 -- 172.21.15.105:0/2497702377 --> v1:172.21.15.105:6812/1313742696 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f7b1003de50 con 0x7f7b04052190 2024-09-21T01:46:10.802 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.796+0000 7f7afffff640 1 -- 172.21.15.105:0/2497702377 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_get_version_reply(handle=1 version=19) ==== 24+0+0 (secure 0 0 0) 0x7f7b10084050 con 0x7f7b18131eb0 2024-09-21T01:46:10.803 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.797+0000 7f21f17fa640 1 -- 172.21.15.105:0/4291667580 <== osd.2 v1:172.21.15.105:6804/2159605016 1 ==== command_reply(tid 1: 0 ) ==== 8+0+26118 (unknown 0 0 1686565167) 0x7f21e8009e50 con 0x7f21d804dd30 2024-09-21T01:46:10.803 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.797+0000 7f7afffff640 1 -- 172.21.15.105:0/2497702377 <== osd.1 v1:172.21.15.105:6812/1313742696 1 ==== command_reply(tid 1: 0 ) ==== 8+0+26118 (unknown 0 0 1686565167) 0x7f7b1003de50 con 0x7f7b04052190 2024-09-21T01:46:10.811 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.810+0000 7f7b1e092640 1 -- 172.21.15.105:0/2497702377 --> v1:172.21.15.105:6812/1313742696 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f7b18128fa0 con 0x7f7b04052190 2024-09-21T01:46:10.812 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.811+0000 7f7afffff640 1 -- 172.21.15.105:0/2497702377 <== osd.1 v1:172.21.15.105:6812/1313742696 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 3077695196) 0x7f7b18128fa0 con 0x7f7b04052190 2024-09-21T01:46:10.813 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.811+0000 7f7b1e092640 1 -- 172.21.15.105:0/2497702377 >> v1:172.21.15.105:6812/1313742696 conn(0x7f7b04052190 legacy=0x7f7b04054620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.813 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.811+0000 7f7b1e092640 1 -- 172.21.15.105:0/2497702377 >> v1:172.21.15.105:6816/3933313069 conn(0x7f7b0404ba50 legacy=0x7f7b0404df60 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:46:10.813 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.811+0000 7f7b1e092640 1 -- 172.21.15.105:0/2497702377 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f7b18131eb0 msgr2=0x7f7b181c4460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.813 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.811+0000 7f7b1e092640 1 --2- 172.21.15.105:0/2497702377 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f7b18131eb0 0x7f7b181c4460 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f7b18072eb0 tx=0x7f7b10033c70 comp rx=0 tx=0).stop 2024-09-21T01:46:10.813 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.811+0000 7f7b17fff640 1 -- 172.21.15.105:0/2497702377 reap_dead start 2024-09-21T01:46:10.813 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.812+0000 7f7b1e092640 1 -- 172.21.15.105:0/2497702377 shutdown_connections 2024-09-21T01:46:10.813 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.812+0000 7f7b1e092640 1 -- 172.21.15.105:0/2497702377 >> 172.21.15.105:0/2497702377 conn(0x7f7b1809f000 msgr2=0x7f7b1809af10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:10.814 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.812+0000 7f7b1e092640 1 -- 172.21.15.105:0/2497702377 shutdown_connections 2024-09-21T01:46:10.814 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.812+0000 7f7b1e092640 1 -- 172.21.15.105:0/2497702377 wait complete. 2024-09-21T01:46:10.824 INFO:teuthology.orchestra.run.smithi105.stdout:38654705683 2024-09-21T01:46:10.824 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T01:46:10.825 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.824+0000 7f21fac20640 1 -- 172.21.15.105:0/4291667580 --> v1:172.21.15.105:6804/2159605016 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f21f4130d90 con 0x7f21d804dd30 2024-09-21T01:46:10.825 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.824+0000 7f21f17fa640 1 -- 172.21.15.105:0/4291667580 <== osd.2 v1:172.21.15.105:6804/2159605016 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 3077695196) 0x7f21f4130d90 con 0x7f21d804dd30 2024-09-21T01:46:10.826 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.824+0000 7f21fac20640 1 -- 172.21.15.105:0/4291667580 >> v1:172.21.15.105:6804/2159605016 conn(0x7f21d804dd30 legacy=0x7f21d80501c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.827 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.824+0000 7f21fac20640 1 -- 172.21.15.105:0/4291667580 >> v1:172.21.15.105:6816/3933313069 conn(0x7f21d80475f0 legacy=0x7f21d8049b00 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:46:10.827 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.824+0000 7f21fac20640 1 -- 172.21.15.105:0/4291667580 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f21f40716c0 msgr2=0x7f21f409b4b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.827 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.824+0000 7f21fac20640 1 --2- 172.21.15.105:0/4291667580 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f21f40716c0 0x7f21f409b4b0 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f21e800c430 tx=0x7f21e800c8f0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.827 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.824+0000 7f21f91bd640 1 -- 172.21.15.105:0/4291667580 reap_dead start 2024-09-21T01:46:10.827 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.825+0000 7f21fac20640 1 -- 172.21.15.105:0/4291667580 shutdown_connections 2024-09-21T01:46:10.827 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.825+0000 7f21fac20640 1 -- 172.21.15.105:0/4291667580 >> 172.21.15.105:0/4291667580 conn(0x7f21f40a1e40 msgr2=0x7f21f4129560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:10.827 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.825+0000 7f21fac20640 1 -- 172.21.15.105:0/4291667580 shutdown_connections 2024-09-21T01:46:10.827 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.825+0000 7f21fac20640 1 -- 172.21.15.105:0/4291667580 wait complete. 2024-09-21T01:46:10.841 INFO:teuthology.orchestra.run.smithi105.stdout:38654705683 2024-09-21T01:46:10.842 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T01:46:10.843 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.840+0000 7f4913189640 1 -- 172.21.15.105:0/1723947833 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f490c094cc0 msgr2=0x7f490c0965d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.843 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.840+0000 7f4913189640 1 --2- 172.21.15.105:0/1723947833 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f490c094cc0 0x7f490c0965d0 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f48f8009830 tx=0x7f48f802a360 comp rx=0 tx=0).stop 2024-09-21T01:46:10.843 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.840+0000 7f4913189640 1 -- 172.21.15.105:0/1723947833 shutdown_connections 2024-09-21T01:46:10.843 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.840+0000 7f4913189640 1 --2- 172.21.15.105:0/1723947833 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f490c139a70 0x7f490c13be90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.843 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.840+0000 7f4913189640 1 --2- 172.21.15.105:0/1723947833 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f490c094cc0 0x7f490c0965d0 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.843 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.840+0000 7f4913189640 1 -- 172.21.15.105:0/1723947833 >> 172.21.15.105:0/1723947833 conn(0x7f490c09e6f0 msgr2=0x7f490c09abc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:10.843 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.840+0000 7f4913189640 1 -- 172.21.15.105:0/1723947833 shutdown_connections 2024-09-21T01:46:10.844 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.840+0000 7f4913189640 1 -- 172.21.15.105:0/1723947833 wait complete. 2024-09-21T01:46:10.845 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.842+0000 7f4913189640 1 Processor -- start 2024-09-21T01:46:10.845 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.842+0000 7f4913189640 1 -- start start 2024-09-21T01:46:10.845 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.843+0000 7f4913189640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f490c094cc0 0x7f490c1ca160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:10.845 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.843+0000 7f4913189640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f490c096b10 0x7f490c1ca6a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:10.845 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.843+0000 7f4913189640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f490c139a70 0x7f490c1cabe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:10.845 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.843+0000 7f4910f25640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f490c094cc0 0x7f490c1ca160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:10.845 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.843+0000 7f4910f25640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f490c094cc0 0x7f490c1ca160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:59884/0 (socket says 172.21.15.105:59884) 2024-09-21T01:46:10.846 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.844+0000 7f490bfff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f490c096b10 0x7f490c1ca6a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:10.846 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.844+0000 7f490bfff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f490c096b10 0x7f490c1ca6a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:53628/0 (socket says 172.21.15.105:53628) 2024-09-21T01:46:10.846 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.844+0000 7f4913189640 1 -- 172.21.15.105:0/2149516108 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f490c13cd10 con 0x7f490c139a70 2024-09-21T01:46:10.847 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.844+0000 7f4911726640 1 --2- 172.21.15.105:0/2149516108 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f490c139a70 0x7f490c1cabe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:10.847 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.844+0000 7f4910f25640 1 -- 172.21.15.105:0/2149516108 learned_addr learned my addr 172.21.15.105:0/2149516108 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:10.847 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.844+0000 7f4913189640 1 -- 172.21.15.105:0/2149516108 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f490c13cb90 con 0x7f490c094cc0 2024-09-21T01:46:10.847 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.844+0000 7f4913189640 1 -- 172.21.15.105:0/2149516108 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f490c13ce90 con 0x7f490c096b10 2024-09-21T01:46:10.847 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.844+0000 7f4910f25640 1 -- 172.21.15.105:0/2149516108 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f490c096b10 msgr2=0x7f490c1ca6a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.847 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.844+0000 7f4910f25640 1 --2- 172.21.15.105:0/2149516108 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f490c096b10 0x7f490c1ca6a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.847 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.844+0000 7f4910f25640 1 -- 172.21.15.105:0/2149516108 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f490c139a70 msgr2=0x7f490c1cabe0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.847 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.844+0000 7f4910f25640 1 --2- 172.21.15.105:0/2149516108 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f490c139a70 0x7f490c1cabe0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.847 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.844+0000 7f4910f25640 1 -- 172.21.15.105:0/2149516108 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f490c123bc0 con 0x7f490c094cc0 2024-09-21T01:46:10.848 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.845+0000 7f4910f25640 1 --2- 172.21.15.105:0/2149516108 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f490c094cc0 0x7f490c1ca160 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f48f8009780 tx=0x7f48f8033c20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:10.848 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.846+0000 7f49097fa640 1 -- 172.21.15.105:0/2149516108 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f48f803f020 con 0x7f490c094cc0 2024-09-21T01:46:10.848 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.846+0000 7f4a282f6640 1 -- 172.21.15.105:0/1324086742 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4a08003a70 msgr2=0x7f4a08003ee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.848 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.846+0000 7f4a282f6640 1 --2- 172.21.15.105:0/1324086742 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4a08003a70 0x7f4a08003ee0 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f4a10002230 tx=0x7f4a08005830 comp rx=0 tx=0).stop 2024-09-21T01:46:10.848 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.847+0000 7f49097fa640 1 -- 172.21.15.105:0/2149516108 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f48f8033e10 con 0x7f490c094cc0 2024-09-21T01:46:10.848 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.847+0000 7f49097fa640 1 -- 172.21.15.105:0/2149516108 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f48f8037040 con 0x7f490c094cc0 2024-09-21T01:46:10.849 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.847+0000 7f4a26893640 1 -- 172.21.15.105:0/1324086742 reap_dead start 2024-09-21T01:46:10.849 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.847+0000 7f4913189640 1 -- 172.21.15.105:0/2149516108 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f490c073050 con 0x7f490c094cc0 2024-09-21T01:46:10.849 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.847+0000 7f4913189640 1 -- 172.21.15.105:0/2149516108 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f490c0734f0 con 0x7f490c094cc0 2024-09-21T01:46:10.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.847+0000 7f4a282f6640 1 -- 172.21.15.105:0/1324086742 shutdown_connections 2024-09-21T01:46:10.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.847+0000 7f4a282f6640 1 -- 172.21.15.105:0/1324086742 >> 172.21.15.105:0/1324086742 conn(0x7f4a2009ed80 msgr2=0x7f4a2009f190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:10.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.848+0000 7f4913189640 1 -- 172.21.15.105:0/2149516108 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f490c0965d0 con 0x7f490c094cc0 2024-09-21T01:46:10.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.848+0000 7f4a282f6640 1 -- 172.21.15.105:0/1324086742 shutdown_connections 2024-09-21T01:46:10.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.848+0000 7f4a282f6640 1 -- 172.21.15.105:0/1324086742 wait complete. 2024-09-21T01:46:10.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.849+0000 7f49097fa640 1 -- 172.21.15.105:0/2149516108 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7f48f8042020 con 0x7f490c094cc0 2024-09-21T01:46:10.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.849+0000 7f490bfff640 1 -- 172.21.15.105:0/2149516108 >> v1:172.21.15.105:6816/3933313069 conn(0x7f48f00475f0 legacy=0x7f48f0049b00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.105:6816/3933313069 2024-09-21T01:46:10.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.850+0000 7f49097fa640 1 -- 172.21.15.105:0/2149516108 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7f48f807d430 con 0x7f490c094cc0 2024-09-21T01:46:10.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.850+0000 7f49097fa640 1 -- 172.21.15.105:0/2149516108 --> v1:172.21.15.105:6808/2514115267 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f48f8046e50 con 0x7f48f004dd30 2024-09-21T01:46:10.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.850+0000 7f4a282f6640 1 Processor -- start 2024-09-21T01:46:10.852 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.850+0000 7f49097fa640 1 -- 172.21.15.105:0/2149516108 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_get_version_reply(handle=1 version=19) ==== 24+0+0 (secure 0 0 0) 0x7f48f807d820 con 0x7f490c094cc0 2024-09-21T01:46:10.852 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.850+0000 7f4a282f6640 1 -- start start 2024-09-21T01:46:10.852 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.851+0000 7f4a282f6640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4a08003a70 0x7f4a2012c5c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:10.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.851+0000 7f4a282f6640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4a2012cb00 0x7f4a2012cf10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:10.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.851+0000 7f4a282f6640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4a201c0f90 0x7f4a201c13a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:10.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.851+0000 7f4a26092640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4a08003a70 0x7f4a2012c5c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:10.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.851+0000 7f4a26092640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4a08003a70 0x7f4a2012c5c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:53634/0 (socket says 172.21.15.105:53634) 2024-09-21T01:46:10.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.851+0000 7f4a26092640 1 -- 172.21.15.105:0/1175696859 learned_addr learned my addr 172.21.15.105:0/1175696859 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:10.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.851+0000 7f4a282f6640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f4a2006f7d0 con 0x7f4a201c0f90 2024-09-21T01:46:10.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.851+0000 7f4a25891640 1 --2- 172.21.15.105:0/1175696859 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4a2012cb00 0x7f4a2012cf10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:10.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.851+0000 7f4a282f6640 1 -- 172.21.15.105:0/1175696859 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f4a2006f650 con 0x7f4a2012cb00 2024-09-21T01:46:10.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.851+0000 7f4a282f6640 1 -- 172.21.15.105:0/1175696859 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f4a2006f950 con 0x7f4a08003a70 2024-09-21T01:46:10.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.851+0000 7f4a26893640 1 --2- 172.21.15.105:0/1175696859 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4a201c0f90 0x7f4a201c13a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:10.854 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.852+0000 7f4a26893640 1 -- 172.21.15.105:0/1175696859 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4a08003a70 msgr2=0x7f4a2012c5c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.854 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.852+0000 7f4a26893640 1 --2- 172.21.15.105:0/1175696859 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4a08003a70 0x7f4a2012c5c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.854 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.852+0000 7f4a26893640 1 -- 172.21.15.105:0/1175696859 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4a2012cb00 msgr2=0x7f4a2012cf10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.855 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.852+0000 7f4a26893640 1 --2- 172.21.15.105:0/1175696859 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4a2012cb00 0x7f4a2012cf10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.855 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.852+0000 7f4a26893640 1 -- 172.21.15.105:0/1175696859 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4a2012e780 con 0x7f4a201c0f90 2024-09-21T01:46:10.855 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.852+0000 7f49097fa640 1 -- 172.21.15.105:0/2149516108 <== osd.3 v1:172.21.15.105:6808/2514115267 1 ==== command_reply(tid 1: 0 ) ==== 8+0+26118 (unknown 0 0 1686565167) 0x7f48f8046e50 con 0x7f48f004dd30 2024-09-21T01:46:10.855 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.852+0000 7f4a26893640 1 --2- 172.21.15.105:0/1175696859 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4a201c0f90 0x7f4a201c13a0 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f4a2009c780 tx=0x7f4a18009740 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:10.856 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.852+0000 7f4a1effd640 1 -- 172.21.15.105:0/1175696859 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a18004440 con 0x7f4a201c0f90 2024-09-21T01:46:10.856 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.852+0000 7f4a1effd640 1 -- 172.21.15.105:0/1175696859 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f4a180035f0 con 0x7f4a201c0f90 2024-09-21T01:46:10.856 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.852+0000 7f4a1effd640 1 -- 172.21.15.105:0/1175696859 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4a18003a20 con 0x7f4a201c0f90 2024-09-21T01:46:10.856 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.852+0000 7f4a282f6640 1 -- 172.21.15.105:0/1175696859 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4a2008c3c0 con 0x7f4a201c0f90 2024-09-21T01:46:10.856 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.852+0000 7f4a282f6640 1 -- 172.21.15.105:0/1175696859 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4a201c1ac0 con 0x7f4a201c0f90 2024-09-21T01:46:10.856 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.853+0000 7f4a1cff9640 1 -- 172.21.15.105:0/1175696859 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f4a04001030 con 0x7f4a201c0f90 2024-09-21T01:46:10.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.855+0000 7f4a1effd640 1 -- 172.21.15.105:0/1175696859 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7f4a18003bc0 con 0x7f4a201c0f90 2024-09-21T01:46:10.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.855+0000 7f4a1effd640 1 -- 172.21.15.105:0/1175696859 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7f4a18065fd0 con 0x7f4a201c0f90 2024-09-21T01:46:10.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.855+0000 7f4a1effd640 1 -- 172.21.15.105:0/1175696859 --> v1:172.21.15.105:6800/4053719786 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f4a0c051a10 con 0x7f4a0c04dd10 2024-09-21T01:46:10.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.855+0000 7f4a26092640 1 -- 172.21.15.105:0/1175696859 >> v1:172.21.15.105:6816/3933313069 conn(0x7f4a0c047640 legacy=0x7f4a0c049b50 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.105:6816/3933313069 2024-09-21T01:46:10.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.855+0000 7f4a1effd640 1 -- 172.21.15.105:0/1175696859 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_get_version_reply(handle=1 version=19) ==== 24+0+0 (secure 0 0 0) 0x7f4a18066380 con 0x7f4a201c0f90 2024-09-21T01:46:10.858 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.856+0000 7f4a1effd640 1 -- 172.21.15.105:0/1175696859 <== osd.0 v1:172.21.15.105:6800/4053719786 1 ==== command_reply(tid 1: 0 ) ==== 8+0+26118 (unknown 0 0 1686565167) 0x7f4a0c051a10 con 0x7f4a0c04dd10 2024-09-21T01:46:10.879 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.877+0000 7f4a1cff9640 1 -- 172.21.15.105:0/1175696859 --> v1:172.21.15.105:6800/4053719786 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f4a04002dd0 con 0x7f4a0c04dd10 2024-09-21T01:46:10.879 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.878+0000 7f4a1effd640 1 -- 172.21.15.105:0/1175696859 <== osd.0 v1:172.21.15.105:6800/4053719786 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 1673232951) 0x7f4a04002dd0 con 0x7f4a0c04dd10 2024-09-21T01:46:10.880 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.878+0000 7f4a1cff9640 1 -- 172.21.15.105:0/1175696859 >> v1:172.21.15.105:6800/4053719786 conn(0x7f4a0c04dd10 legacy=0x7f4a0c0501a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.880 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.878+0000 7f4a1cff9640 1 -- 172.21.15.105:0/1175696859 >> v1:172.21.15.105:6816/3933313069 conn(0x7f4a0c047640 legacy=0x7f4a0c049b50 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:46:10.881 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.878+0000 7f4a1cff9640 1 -- 172.21.15.105:0/1175696859 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4a201c0f90 msgr2=0x7f4a201c13a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.881 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.878+0000 7f4a1cff9640 1 --2- 172.21.15.105:0/1175696859 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4a201c0f90 0x7f4a201c13a0 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f4a2009c780 tx=0x7f4a18009740 comp rx=0 tx=0).stop 2024-09-21T01:46:10.881 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.879+0000 7f4a26893640 1 -- 172.21.15.105:0/1175696859 reap_dead start 2024-09-21T01:46:10.881 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.879+0000 7f4a1cff9640 1 -- 172.21.15.105:0/1175696859 shutdown_connections 2024-09-21T01:46:10.881 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.879+0000 7f4a1cff9640 1 -- 172.21.15.105:0/1175696859 >> 172.21.15.105:0/1175696859 conn(0x7f4a2009ed80 msgr2=0x7f4a20125a30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:10.881 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.879+0000 7f4a1cff9640 1 -- 172.21.15.105:0/1175696859 shutdown_connections 2024-09-21T01:46:10.881 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.879+0000 7f4a1cff9640 1 -- 172.21.15.105:0/1175696859 wait complete. 2024-09-21T01:46:10.881 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.879+0000 7f4913189640 1 -- 172.21.15.105:0/2149516108 --> v1:172.21.15.105:6808/2514115267 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f490c0741e0 con 0x7f48f004dd30 2024-09-21T01:46:10.881 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.880+0000 7f49097fa640 1 -- 172.21.15.105:0/2149516108 <== osd.3 v1:172.21.15.105:6808/2514115267 2 ==== command_reply(tid 2: 0 ) ==== 8+0+11 (unknown 0 0 3077695196) 0x7f490c0741e0 con 0x7f48f004dd30 2024-09-21T01:46:10.883 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.880+0000 7f4913189640 1 -- 172.21.15.105:0/2149516108 >> v1:172.21.15.105:6808/2514115267 conn(0x7f48f004dd30 legacy=0x7f48f00501c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.883 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.881+0000 7f4913189640 1 -- 172.21.15.105:0/2149516108 >> v1:172.21.15.105:6816/3933313069 conn(0x7f48f00475f0 legacy=0x7f48f0049b00 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:46:10.883 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.881+0000 7f4913189640 1 -- 172.21.15.105:0/2149516108 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f490c094cc0 msgr2=0x7f490c1ca160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.883 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.881+0000 7f4913189640 1 --2- 172.21.15.105:0/2149516108 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f490c094cc0 0x7f490c1ca160 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f48f8009780 tx=0x7f48f8033c20 comp rx=0 tx=0).stop 2024-09-21T01:46:10.883 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.881+0000 7f4911726640 1 -- 172.21.15.105:0/2149516108 reap_dead start 2024-09-21T01:46:10.883 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.881+0000 7f4913189640 1 -- 172.21.15.105:0/2149516108 shutdown_connections 2024-09-21T01:46:10.883 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.881+0000 7f4913189640 1 -- 172.21.15.105:0/2149516108 >> 172.21.15.105:0/2149516108 conn(0x7f490c09e6f0 msgr2=0x7f490c12f810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:10.883 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.881+0000 7f4913189640 1 -- 172.21.15.105:0/2149516108 shutdown_connections 2024-09-21T01:46:10.883 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.881+0000 7f4913189640 1 -- 172.21.15.105:0/2149516108 wait complete. 2024-09-21T01:46:10.894 INFO:teuthology.orchestra.run.smithi105.stdout:38654705684 2024-09-21T01:46:10.895 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T01:46:10.901 INFO:teuthology.orchestra.run.smithi105.stdout:38654705683 2024-09-21T01:46:10.902 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T01:46:10.989 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.987+0000 7fe85b486640 1 -- 172.21.15.105:0/3277945728 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe85406cd90 msgr2=0x7fe8540965d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.989 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.987+0000 7fe84bfff640 1 -- 172.21.15.105:0/3277945728 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe850035020 con 0x7fe85406cd90 2024-09-21T01:46:10.989 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.987+0000 7fe85b486640 1 --2- 172.21.15.105:0/3277945728 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe85406cd90 0x7fe8540965d0 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7fe850005fd0 tx=0x7fe85002a420 comp rx=0 tx=0).stop 2024-09-21T01:46:10.989 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.987+0000 7fe85b486640 1 -- 172.21.15.105:0/3277945728 shutdown_connections 2024-09-21T01:46:10.989 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.987+0000 7fe85b486640 1 --2- 172.21.15.105:0/3277945728 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fe854134270 0x7fe854136690 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.989 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.987+0000 7fe85b486640 1 --2- 172.21.15.105:0/3277945728 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fe854096b10 0x7fe854096f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.989 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.987+0000 7fe85b486640 1 --2- 172.21.15.105:0/3277945728 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe85406cd90 0x7fe8540965d0 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.989 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.987+0000 7fe85b486640 1 -- 172.21.15.105:0/3277945728 >> 172.21.15.105:0/3277945728 conn(0x7fe85409e6f0 msgr2=0x7fe85409abc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:10.989 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.987+0000 7fe85b486640 1 -- 172.21.15.105:0/3277945728 shutdown_connections 2024-09-21T01:46:10.990 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.987+0000 7fe85b486640 1 -- 172.21.15.105:0/3277945728 wait complete. 2024-09-21T01:46:10.991 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.989+0000 7fe85b486640 1 Processor -- start 2024-09-21T01:46:10.991 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.989+0000 7fe85b486640 1 -- start start 2024-09-21T01:46:10.992 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.991+0000 7fe85b486640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fe854096b10 0x7fe8541c5b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:10.992 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.991+0000 7fe85b486640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fe854134270 0x7fe8541c6060 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:10.993 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.991+0000 7fe858a21640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fe854134270 0x7fe8541c6060 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:10.993 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.991+0000 7fe859222640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fe854096b10 0x7fe8541c5b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:10.993 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.991+0000 7fe859222640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fe854096b10 0x7fe8541c5b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:53670/0 (socket says 172.21.15.105:53670) 2024-09-21T01:46:10.993 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.991+0000 7fe858a21640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fe854134270 0x7fe8541c6060 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:59912/0 (socket says 172.21.15.105:59912) 2024-09-21T01:46:10.993 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.991+0000 7fe85b486640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe8541c65a0 0x7fe854073530 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:10.993 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.991+0000 7fe858a21640 1 -- 172.21.15.105:0/1496749831 learned_addr learned my addr 172.21.15.105:0/1496749831 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:10.993 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.991+0000 7fe85b486640 1 -- 172.21.15.105:0/1496749831 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7fe854137690 con 0x7fe8541c65a0 2024-09-21T01:46:10.994 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.991+0000 7fe859a23640 1 --2- 172.21.15.105:0/1496749831 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe8541c65a0 0x7fe854073530 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:10.994 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.992+0000 7fe85b486640 1 -- 172.21.15.105:0/1496749831 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7fe854137510 con 0x7fe854134270 2024-09-21T01:46:10.994 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.992+0000 7fe85b486640 1 -- 172.21.15.105:0/1496749831 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7fe854137810 con 0x7fe854096b10 2024-09-21T01:46:10.995 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.992+0000 7fe859a23640 1 -- 172.21.15.105:0/1496749831 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fe854096b10 msgr2=0x7fe8541c5b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.995 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.992+0000 7fe859a23640 1 --2- 172.21.15.105:0/1496749831 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fe854096b10 0x7fe8541c5b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.995 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.992+0000 7fe859a23640 1 -- 172.21.15.105:0/1496749831 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fe854134270 msgr2=0x7fe8541c6060 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:10.995 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.993+0000 7fe859a23640 1 --2- 172.21.15.105:0/1496749831 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fe854134270 0x7fe8541c6060 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:10.995 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.993+0000 7fe859a23640 1 -- 172.21.15.105:0/1496749831 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe854073270 con 0x7fe8541c65a0 2024-09-21T01:46:10.995 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.993+0000 7fe859a23640 1 --2- 172.21.15.105:0/1496749831 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe8541c65a0 0x7fe854073530 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7fe84c00e9b0 tx=0x7fe84c00ee70 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:10.995 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.993+0000 7fe858a21640 1 --2- 172.21.15.105:0/1496749831 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fe854134270 0x7fe8541c6060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T01:46:10.996 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.993+0000 7fe849ffb640 1 -- 172.21.15.105:0/1496749831 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe84c004d20 con 0x7fe8541c65a0 2024-09-21T01:46:10.996 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.994+0000 7fe849ffb640 1 -- 172.21.15.105:0/1496749831 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fe84c00cb20 con 0x7fe8541c65a0 2024-09-21T01:46:10.996 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.994+0000 7fe849ffb640 1 -- 172.21.15.105:0/1496749831 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe84c005020 con 0x7fe8541c65a0 2024-09-21T01:46:10.996 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.994+0000 7fe85b486640 1 -- 172.21.15.105:0/1496749831 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe854072060 con 0x7fe8541c65a0 2024-09-21T01:46:10.996 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.995+0000 7fe85b486640 1 -- 172.21.15.105:0/1496749831 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe854073a70 con 0x7fe8541c65a0 2024-09-21T01:46:10.997 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.996+0000 7fe849ffb640 1 -- 172.21.15.105:0/1496749831 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7fe84c0051c0 con 0x7fe8541c65a0 2024-09-21T01:46:10.998 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.997+0000 7fe859222640 1 -- 172.21.15.105:0/1496749831 >> v1:172.21.15.105:6816/3933313069 conn(0x7fe838047640 legacy=0x7fe838049b50 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.105:6816/3933313069 2024-09-21T01:46:11.002 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:10.997+0000 7fe85b486640 1 -- 172.21.15.105:0/1496749831 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fe8540965d0 con 0x7fe8541c65a0 2024-09-21T01:46:11.002 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.000+0000 7fe849ffb640 1 -- 172.21.15.105:0/1496749831 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7fe84c05e600 con 0x7fe8541c65a0 2024-09-21T01:46:11.006 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.004+0000 7fe849ffb640 1 -- 172.21.15.105:0/1496749831 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7fe84c03c640 con 0x7fe8541c65a0 2024-09-21T01:46:11.028 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.026+0000 7f117e95e640 1 -- 172.21.15.105:0/35122557 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f11780945b0 msgr2=0x7f11780949c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:11.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.026+0000 7f117e95e640 1 --2- 172.21.15.105:0/35122557 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f11780945b0 0x7f11780949c0 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f1170009ca0 tx=0x7f117002aa50 comp rx=0 tx=0).stop 2024-09-21T01:46:11.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.027+0000 7f117e95e640 1 -- 172.21.15.105:0/35122557 shutdown_connections 2024-09-21T01:46:11.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.027+0000 7f117e95e640 1 --2- 172.21.15.105:0/35122557 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f11780945b0 0x7f11780949c0 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.027+0000 7f117e95e640 1 --2- 172.21.15.105:0/35122557 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f1178098030 0x7f1178094070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.027+0000 7f117e95e640 1 --2- 172.21.15.105:0/35122557 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f11780976e0 0x7f1178097af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.027+0000 7f117e95e640 1 -- 172.21.15.105:0/35122557 >> 172.21.15.105:0/35122557 conn(0x7f117809e6f0 msgr2=0x7f117809abc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:11.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.027+0000 7f117e95e640 1 -- 172.21.15.105:0/35122557 shutdown_connections 2024-09-21T01:46:11.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.027+0000 7f117e95e640 1 -- 172.21.15.105:0/35122557 wait complete. 2024-09-21T01:46:11.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.028+0000 7f117e95e640 1 Processor -- start 2024-09-21T01:46:11.030 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.029+0000 7f117e95e640 1 -- start start 2024-09-21T01:46:11.031 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.030+0000 7f117e95e640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f11780945b0 0x7f1178130c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:11.032 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.030+0000 7f1177fff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f11780945b0 0x7f1178130c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:11.032 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.030+0000 7f1177fff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f11780945b0 0x7f1178130c90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:59928/0 (socket says 172.21.15.105:59928) 2024-09-21T01:46:11.032 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.030+0000 7f1177fff640 1 -- 172.21.15.105:0/1046013041 learned_addr learned my addr 172.21.15.105:0/1046013041 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:11.032 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.030+0000 7f117e95e640 1 --2- 172.21.15.105:0/1046013041 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f11780976e0 0x7f11781311d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:11.033 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.030+0000 7f117e95e640 1 --2- 172.21.15.105:0/1046013041 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1178098030 0x7f1178131710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:11.033 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.031+0000 7f11777fe640 1 --2- 172.21.15.105:0/1046013041 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f11780976e0 0x7f11781311d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:11.033 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.031+0000 7f117cefb640 1 --2- 172.21.15.105:0/1046013041 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1178098030 0x7f1178131710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:11.033 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.031+0000 7f117e95e640 1 -- 172.21.15.105:0/1046013041 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f1178096cc0 con 0x7f1178098030 2024-09-21T01:46:11.033 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.031+0000 7f117e95e640 1 -- 172.21.15.105:0/1046013041 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f1178096b40 con 0x7f11780945b0 2024-09-21T01:46:11.033 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.031+0000 7f117e95e640 1 -- 172.21.15.105:0/1046013041 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f1178096e40 con 0x7f11780976e0 2024-09-21T01:46:11.034 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.032+0000 7f11777fe640 1 -- 172.21.15.105:0/1046013041 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f11780945b0 msgr2=0x7f1178130c90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:11.034 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.032+0000 7f11777fe640 1 --2- 172.21.15.105:0/1046013041 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f11780945b0 0x7f1178130c90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.035 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.032+0000 7f11777fe640 1 -- 172.21.15.105:0/1046013041 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1178098030 msgr2=0x7f1178131710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:11.035 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.032+0000 7f11777fe640 1 --2- 172.21.15.105:0/1046013041 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1178098030 0x7f1178131710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.035 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.032+0000 7f11777fe640 1 -- 172.21.15.105:0/1046013041 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f1178123ac0 con 0x7f11780976e0 2024-09-21T01:46:11.035 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.032+0000 7f1177fff640 1 --2- 172.21.15.105:0/1046013041 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f11780945b0 0x7f1178130c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:11.035 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.032+0000 7f11777fe640 1 --2- 172.21.15.105:0/1046013041 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f11780976e0 0x7f11781311d0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f116c00ea90 tx=0x7f116c00ef50 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:11.035 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.032+0000 7f117cefb640 1 --2- 172.21.15.105:0/1046013041 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1178098030 0x7f1178131710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T01:46:11.035 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.033+0000 7f1174ff9640 1 -- 172.21.15.105:0/1046013041 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f116c00cc90 con 0x7f11780976e0 2024-09-21T01:46:11.035 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.033+0000 7f1174ff9640 1 -- 172.21.15.105:0/1046013041 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f116c004020 con 0x7f11780976e0 2024-09-21T01:46:11.036 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.034+0000 7f117e95e640 1 -- 172.21.15.105:0/1046013041 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1178123cf0 con 0x7f11780976e0 2024-09-21T01:46:11.036 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.034+0000 7f117e95e640 1 -- 172.21.15.105:0/1046013041 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f11781c9e40 con 0x7f11780976e0 2024-09-21T01:46:11.036 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.034+0000 7f1174ff9640 1 -- 172.21.15.105:0/1046013041 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f116c004450 con 0x7f11780976e0 2024-09-21T01:46:11.036 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.035+0000 7f117e95e640 1 -- 172.21.15.105:0/1046013041 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f11780973d0 con 0x7f11780976e0 2024-09-21T01:46:11.037 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.036+0000 7f1174ff9640 1 -- 172.21.15.105:0/1046013041 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7f116c0035f0 con 0x7f11780976e0 2024-09-21T01:46:11.038 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.036+0000 7f1174ff9640 1 -- 172.21.15.105:0/1046013041 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7f116c05d1f0 con 0x7f11780976e0 2024-09-21T01:46:11.038 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.037+0000 7f1177fff640 1 -- 172.21.15.105:0/1046013041 >> v1:172.21.15.105:6816/3933313069 conn(0x7f1164047670 legacy=0x7f1164049b80 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.105:6816/3933313069 2024-09-21T01:46:11.045 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.044+0000 7f1174ff9640 1 -- 172.21.15.105:0/1046013041 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7f116c03c940 con 0x7f11780976e0 2024-09-21T01:46:11.093 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.091+0000 7f06f9c07640 1 -- 172.21.15.105:0/1153291778 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f06f40945b0 msgr2=0x7f06f40949c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:11.093 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.091+0000 7f06f9c07640 1 --2- 172.21.15.105:0/1153291778 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f06f40945b0 0x7f06f40949c0 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f06ec005fd0 tx=0x7f06ec02a780 comp rx=0 tx=0).stop 2024-09-21T01:46:11.093 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.091+0000 7f06f9c07640 1 -- 172.21.15.105:0/1153291778 shutdown_connections 2024-09-21T01:46:11.093 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.091+0000 7f06f9c07640 1 --2- 172.21.15.105:0/1153291778 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f06f40945b0 0x7f06f40949c0 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.093 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.091+0000 7f06f9c07640 1 --2- 172.21.15.105:0/1153291778 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f06f4098030 0x7f06f4094070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.093 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.091+0000 7f06f9c07640 1 --2- 172.21.15.105:0/1153291778 >> v2:172.21.15.105:3300/0 conn(0x7f06f40976e0 0x7f06f4097af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.093 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.091+0000 7f06f9c07640 1 -- 172.21.15.105:0/1153291778 >> 172.21.15.105:0/1153291778 conn(0x7f06f409e6f0 msgr2=0x7f06f409abc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:11.093 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.091+0000 7f06f9c07640 1 -- 172.21.15.105:0/1153291778 shutdown_connections 2024-09-21T01:46:11.093 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.091+0000 7f06f9c07640 1 -- 172.21.15.105:0/1153291778 wait complete. 2024-09-21T01:46:11.094 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.092+0000 7f06f9c07640 1 Processor -- start 2024-09-21T01:46:11.094 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.093+0000 7f06f9c07640 1 -- start start 2024-09-21T01:46:11.094 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.093+0000 7f06f9c07640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f06f40945b0 0x7f06f4123180 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:11.096 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.093+0000 7f06f9c07640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f06f40976e0 0x7f06f41236c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:11.096 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.093+0000 7f06f9c07640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f06f4098030 0x7f06f4123c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:11.097 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.093+0000 7f06f9c07640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f06f4096d50 con 0x7f06f40945b0 2024-09-21T01:46:11.097 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.093+0000 7f06f9c07640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f06f4096bd0 con 0x7f06f40976e0 2024-09-21T01:46:11.097 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.093+0000 7f06f9c07640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f06f4096ed0 con 0x7f06f4098030 2024-09-21T01:46:11.097 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.094+0000 7f06f37fe640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f06f40945b0 0x7f06f4123180 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:11.097 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.094+0000 7f06f37fe640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f06f40945b0 0x7f06f4123180 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:33524/0 (socket says 172.21.15.105:33524) 2024-09-21T01:46:11.097 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.094+0000 7f06f37fe640 1 -- 172.21.15.105:0/922761459 learned_addr learned my addr 172.21.15.105:0/922761459 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:11.097 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.094+0000 7f06f2ffd640 1 --2- 172.21.15.105:0/922761459 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f06f40976e0 0x7f06f41236c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:11.097 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.094+0000 7f06f37fe640 1 -- 172.21.15.105:0/922761459 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f06f4098030 msgr2=0x7f06f4123c00 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:46:11.097 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.094+0000 7f06f37fe640 1 --2- 172.21.15.105:0/922761459 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f06f4098030 0x7f06f4123c00 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.097 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.094+0000 7f06f37fe640 1 -- 172.21.15.105:0/922761459 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f06f40976e0 msgr2=0x7f06f41236c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:11.097 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.094+0000 7f06f37fe640 1 --2- 172.21.15.105:0/922761459 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f06f40976e0 0x7f06f41236c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.097 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.094+0000 7f06f37fe640 1 -- 172.21.15.105:0/922761459 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f06f4124ec0 con 0x7f06f40945b0 2024-09-21T01:46:11.097 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.094+0000 7f06f2ffd640 1 --2- 172.21.15.105:0/922761459 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f06f40976e0 0x7f06f41236c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T01:46:11.097 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.094+0000 7f06f37fe640 1 --2- 172.21.15.105:0/922761459 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f06f40945b0 0x7f06f4123180 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f06e400ecb0 tx=0x7f06e400c6a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:11.098 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.095+0000 7f06dbfff640 1 -- 172.21.15.105:0/922761459 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f06e4004020 con 0x7f06f40945b0 2024-09-21T01:46:11.098 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.095+0000 7f06dbfff640 1 -- 172.21.15.105:0/922761459 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f06e40041c0 con 0x7f06f40945b0 2024-09-21T01:46:11.098 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.095+0000 7f06dbfff640 1 -- 172.21.15.105:0/922761459 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f06e40035f0 con 0x7f06f40945b0 2024-09-21T01:46:11.098 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.095+0000 7f06f9c07640 1 -- 172.21.15.105:0/922761459 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f06f4125500 con 0x7f06f40945b0 2024-09-21T01:46:11.098 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.095+0000 7f06f9c07640 1 -- 172.21.15.105:0/922761459 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f06f4129f60 con 0x7f06f40945b0 2024-09-21T01:46:11.099 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.097+0000 7f06f9c07640 1 -- 172.21.15.105:0/922761459 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f06f40973f0 con 0x7f06f40945b0 2024-09-21T01:46:11.099 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.097+0000 7f06dbfff640 1 -- 172.21.15.105:0/922761459 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7f06e4003790 con 0x7f06f40945b0 2024-09-21T01:46:11.099 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.098+0000 7f06dbfff640 1 -- 172.21.15.105:0/922761459 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7f06e405d500 con 0x7f06f40945b0 2024-09-21T01:46:11.099 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.098+0000 7f06f2ffd640 1 -- 172.21.15.105:0/922761459 >> v1:172.21.15.105:6816/3933313069 conn(0x7f06e0047620 legacy=0x7f06e0049b30 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.105:6816/3933313069 2024-09-21T01:46:11.103 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.102+0000 7f06dbfff640 1 -- 172.21.15.105:0/922761459 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7f06e405c690 con 0x7f06f40945b0 2024-09-21T01:46:11.132 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.130+0000 7f8ca31e4640 1 -- 172.21.15.105:0/931456776 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8c9c0985f0 msgr2=0x7f8c9c098a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:11.132 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.130+0000 7f8ca31e4640 1 --2- 172.21.15.105:0/931456776 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8c9c0985f0 0x7f8c9c098a00 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f8c94005fd0 tx=0x7f8c9402a870 comp rx=0 tx=0).stop 2024-09-21T01:46:11.133 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.130+0000 7f8ca31e4640 1 -- 172.21.15.105:0/931456776 shutdown_connections 2024-09-21T01:46:11.133 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.130+0000 7f8ca31e4640 1 --2- 172.21.15.105:0/931456776 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8c9c0985f0 0x7f8c9c098a00 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.133 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.130+0000 7f8ca31e4640 1 --2- 172.21.15.105:0/931456776 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8c9c0721b0 0x7f8c9c0980b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.133 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.130+0000 7f8ca31e4640 1 --2- 172.21.15.105:0/931456776 >> v2:172.21.15.105:3300/0 conn(0x7f8c9c071860 0x7f8c9c071c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.133 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.130+0000 7f8ca31e4640 1 -- 172.21.15.105:0/931456776 >> 172.21.15.105:0/931456776 conn(0x7f8c9c09f000 msgr2=0x7f8c9c09b4d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:11.133 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.131+0000 7f8ca31e4640 1 -- 172.21.15.105:0/931456776 shutdown_connections 2024-09-21T01:46:11.133 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.131+0000 7f8ca31e4640 1 -- 172.21.15.105:0/931456776 wait complete. 2024-09-21T01:46:11.134 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.132+0000 7f8ca31e4640 1 Processor -- start 2024-09-21T01:46:11.134 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.132+0000 7f8ca31e4640 1 -- start start 2024-09-21T01:46:11.135 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.134+0000 7f8ca31e4640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c9c071860 0x7f8c9c0970a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:11.136 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.134+0000 7f8ca0f80640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c9c071860 0x7f8c9c0970a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:11.136 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.134+0000 7f8ca0f80640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c9c071860 0x7f8c9c0970a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:50420/0 (socket says 172.21.15.105:50420) 2024-09-21T01:46:11.136 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.134+0000 7f8ca31e4640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8c9c0721b0 0x7f8c9c0975e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:11.136 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.134+0000 7f8c9bfff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8c9c0721b0 0x7f8c9c0975e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:11.136 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.134+0000 7f8c9bfff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8c9c0721b0 0x7f8c9c0975e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:49966/0 (socket says 172.21.15.105:49966) 2024-09-21T01:46:11.136 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.134+0000 7f8ca0f80640 1 -- 172.21.15.105:0/109499419 learned_addr learned my addr 172.21.15.105:0/109499419 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:11.136 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.134+0000 7f8ca31e4640 1 --2- 172.21.15.105:0/109499419 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8c9c0985f0 0x7f8c9c097b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:11.137 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.135+0000 7f8ca1781640 1 --2- 172.21.15.105:0/109499419 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8c9c0985f0 0x7f8c9c097b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:11.137 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.135+0000 7f8ca31e4640 1 -- 172.21.15.105:0/109499419 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f8c9c1341c0 con 0x7f8c9c071860 2024-09-21T01:46:11.137 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.135+0000 7f8ca31e4640 1 -- 172.21.15.105:0/109499419 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f8c9c134040 con 0x7f8c9c0985f0 2024-09-21T01:46:11.137 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.135+0000 7f8ca31e4640 1 -- 172.21.15.105:0/109499419 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f8c9c134340 con 0x7f8c9c0721b0 2024-09-21T01:46:11.138 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.136+0000 7f8c9bfff640 1 -- 172.21.15.105:0/109499419 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8c9c0985f0 msgr2=0x7f8c9c097b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:11.138 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.136+0000 7f8c9bfff640 1 --2- 172.21.15.105:0/109499419 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8c9c0985f0 0x7f8c9c097b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.138 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.136+0000 7f8c9bfff640 1 -- 172.21.15.105:0/109499419 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c9c071860 msgr2=0x7f8c9c0970a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:11.138 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.136+0000 7f8c9bfff640 1 --2- 172.21.15.105:0/109499419 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c9c071860 0x7f8c9c0970a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.139 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.136+0000 7f8c9bfff640 1 -- 172.21.15.105:0/109499419 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8c9c1320e0 con 0x7f8c9c0721b0 2024-09-21T01:46:11.139 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.136+0000 7f8ca0f80640 1 --2- 172.21.15.105:0/109499419 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c9c071860 0x7f8c9c0970a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:11.139 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.136+0000 7f8ca1781640 1 --2- 172.21.15.105:0/109499419 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8c9c0985f0 0x7f8c9c097b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:11.139 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.136+0000 7f8c9bfff640 1 --2- 172.21.15.105:0/109499419 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8c9c0721b0 0x7f8c9c0975e0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f8c88005fd0 tx=0x7f8c88009d30 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:11.139 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.137+0000 7f8c997fa640 1 -- 172.21.15.105:0/109499419 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8c88004030 con 0x7f8c9c0721b0 2024-09-21T01:46:11.139 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.137+0000 7f8c997fa640 1 -- 172.21.15.105:0/109499419 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f8c880041d0 con 0x7f8c9c0721b0 2024-09-21T01:46:11.139 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.137+0000 7f8c997fa640 1 -- 172.21.15.105:0/109499419 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8c88007770 con 0x7f8c9c0721b0 2024-09-21T01:46:11.139 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.137+0000 7f8ca31e4640 1 -- 172.21.15.105:0/109499419 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8c9c131780 con 0x7f8c9c0721b0 2024-09-21T01:46:11.139 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.138+0000 7f8ca31e4640 1 -- 172.21.15.105:0/109499419 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f8c9c06ebc0 con 0x7f8c9c0721b0 2024-09-21T01:46:11.141 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.140+0000 7f8ca31e4640 1 -- 172.21.15.105:0/109499419 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8c9c09a260 con 0x7f8c9c0721b0 2024-09-21T01:46:11.141 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.140+0000 7f8c997fa640 1 -- 172.21.15.105:0/109499419 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7f8c88007910 con 0x7f8c9c0721b0 2024-09-21T01:46:11.141 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.140+0000 7f8ca0f80640 1 -- 172.21.15.105:0/109499419 >> v1:172.21.15.105:6816/3933313069 conn(0x7f8c800475f0 legacy=0x7f8c80049b00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.105:6816/3933313069 2024-09-21T01:46:11.143 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.142+0000 7f8c997fa640 1 -- 172.21.15.105:0/109499419 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7f8c88066300 con 0x7f8c9c0721b0 2024-09-21T01:46:11.149 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.148+0000 7f8c997fa640 1 -- 172.21.15.105:0/109499419 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7f8c9c09a260 con 0x7f8c9c0721b0 2024-09-21T01:46:11.199 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.198+0000 7fe859222640 1 -- 172.21.15.105:0/1496749831 >> v1:172.21.15.105:6816/3933313069 conn(0x7fe838047640 legacy=0x7fe838049b50 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.105:6816/3933313069 2024-09-21T01:46:11.239 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.238+0000 7f1177fff640 1 -- 172.21.15.105:0/1046013041 >> v1:172.21.15.105:6816/3933313069 conn(0x7f1164047670 legacy=0x7f1164049b80 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.105:6816/3933313069 2024-09-21T01:46:11.286 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.285+0000 7fe85b486640 1 -- 172.21.15.105:0/1496749831 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7fe85406a820 con 0x7fe8541c65a0 2024-09-21T01:46:11.287 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.285+0000 7fe849ffb640 1 -- 172.21.15.105:0/1496749831 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fe84c03c8a0 con 0x7fe8541c65a0 2024-09-21T01:46:11.287 INFO:teuthology.orchestra.run.smithi105.stdout:38654705681 2024-09-21T01:46:11.289 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.287+0000 7fe85b486640 1 -- 172.21.15.105:0/1496749831 >> v1:172.21.15.105:6816/3933313069 conn(0x7fe838047640 legacy=0x7fe838049b50 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:46:11.289 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.287+0000 7fe85b486640 1 -- 172.21.15.105:0/1496749831 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe8541c65a0 msgr2=0x7fe854073530 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:11.289 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.287+0000 7fe85b486640 1 --2- 172.21.15.105:0/1496749831 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe8541c65a0 0x7fe854073530 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7fe84c00e9b0 tx=0x7fe84c00ee70 comp rx=0 tx=0).stop 2024-09-21T01:46:11.289 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.288+0000 7fe85b486640 1 -- 172.21.15.105:0/1496749831 shutdown_connections 2024-09-21T01:46:11.289 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.288+0000 7fe85b486640 1 --2- 172.21.15.105:0/1496749831 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe8541c65a0 0x7fe854073530 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.289 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.288+0000 7fe85b486640 1 --2- 172.21.15.105:0/1496749831 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fe854134270 0x7fe8541c6060 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.289 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.288+0000 7fe85b486640 1 --2- 172.21.15.105:0/1496749831 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fe854096b10 0x7fe8541c5b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.289 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.288+0000 7fe85b486640 1 -- 172.21.15.105:0/1496749831 >> 172.21.15.105:0/1496749831 conn(0x7fe85409e6f0 msgr2=0x7fe854133a50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:11.289 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.288+0000 7fe85b486640 1 -- 172.21.15.105:0/1496749831 shutdown_connections 2024-09-21T01:46:11.290 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.288+0000 7fe85b486640 1 -- 172.21.15.105:0/1496749831 wait complete. 2024-09-21T01:46:11.301 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.299+0000 7f06f2ffd640 1 -- 172.21.15.105:0/922761459 >> v1:172.21.15.105:6816/3933313069 conn(0x7f06e0047620 legacy=0x7f06e0049b30 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.105:6816/3933313069 2024-09-21T01:46:11.301 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705681 for osd.1 2024-09-21T01:46:11.345 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.343+0000 7f8ca0f80640 1 -- 172.21.15.105:0/109499419 >> v1:172.21.15.105:6816/3933313069 conn(0x7f8c800475f0 legacy=0x7f8c80049b00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.105:6816/3933313069 2024-09-21T01:46:11.349 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.348+0000 7f117e95e640 1 -- 172.21.15.105:0/1046013041 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7f117806a820 con 0x7f11780976e0 2024-09-21T01:46:11.349 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.348+0000 7f1174ff9640 1 -- 172.21.15.105:0/1046013041 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f116c03cba0 con 0x7f11780976e0 2024-09-21T01:46:11.350 INFO:teuthology.orchestra.run.smithi105.stdout:38654705681 2024-09-21T01:46:11.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.351+0000 7f117e95e640 1 -- 172.21.15.105:0/1046013041 >> v1:172.21.15.105:6816/3933313069 conn(0x7f1164047670 legacy=0x7f1164049b80 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:46:11.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.351+0000 7f117e95e640 1 -- 172.21.15.105:0/1046013041 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f11780976e0 msgr2=0x7f11781311d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:11.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.351+0000 7f117e95e640 1 --2- 172.21.15.105:0/1046013041 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f11780976e0 0x7f11781311d0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f116c00ea90 tx=0x7f116c00ef50 comp rx=0 tx=0).stop 2024-09-21T01:46:11.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.351+0000 7f117e95e640 1 -- 172.21.15.105:0/1046013041 shutdown_connections 2024-09-21T01:46:11.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.351+0000 7f117e95e640 1 --2- 172.21.15.105:0/1046013041 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1178098030 0x7f1178131710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.351+0000 7f117e95e640 1 --2- 172.21.15.105:0/1046013041 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f11780976e0 0x7f11781311d0 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.351+0000 7f117e95e640 1 --2- 172.21.15.105:0/1046013041 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f11780945b0 0x7f1178130c90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.356 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.351+0000 7f117e95e640 1 -- 172.21.15.105:0/1046013041 >> 172.21.15.105:0/1046013041 conn(0x7f117809e6f0 msgr2=0x7f117812d000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:11.356 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.351+0000 7f117e95e640 1 -- 172.21.15.105:0/1046013041 shutdown_connections 2024-09-21T01:46:11.356 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.351+0000 7f117e95e640 1 -- 172.21.15.105:0/1046013041 wait complete. 2024-09-21T01:46:11.369 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705681 for osd.2 2024-09-21T01:46:11.406 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.404+0000 7f06f9c07640 1 -- 172.21.15.105:0/922761459 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7f06f406a820 con 0x7f06f40945b0 2024-09-21T01:46:11.406 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.404+0000 7f06dbfff640 1 -- 172.21.15.105:0/922761459 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f06e403ccd0 con 0x7f06f40945b0 2024-09-21T01:46:11.406 INFO:teuthology.orchestra.run.smithi105.stdout:38654705682 2024-09-21T01:46:11.409 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.407+0000 7f06f9c07640 1 -- 172.21.15.105:0/922761459 >> v1:172.21.15.105:6816/3933313069 conn(0x7f06e0047620 legacy=0x7f06e0049b30 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:46:11.409 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.407+0000 7f06f9c07640 1 -- 172.21.15.105:0/922761459 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f06f40945b0 msgr2=0x7f06f4123180 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:11.409 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.407+0000 7f06f9c07640 1 --2- 172.21.15.105:0/922761459 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f06f40945b0 0x7f06f4123180 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7f06e400ecb0 tx=0x7f06e400c6a0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.409 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.407+0000 7f06f9c07640 1 -- 172.21.15.105:0/922761459 shutdown_connections 2024-09-21T01:46:11.409 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.407+0000 7f06f9c07640 1 --2- 172.21.15.105:0/922761459 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f06f4098030 0x7f06f4123c00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.409 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.407+0000 7f06f9c07640 1 --2- 172.21.15.105:0/922761459 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f06f40976e0 0x7f06f41236c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.409 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.407+0000 7f06f9c07640 1 --2- 172.21.15.105:0/922761459 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f06f40945b0 0x7f06f4123180 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.410 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.407+0000 7f06f9c07640 1 -- 172.21.15.105:0/922761459 >> 172.21.15.105:0/922761459 conn(0x7f06f409e6f0 msgr2=0x7f06f41b4a50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:11.410 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.408+0000 7f06f9c07640 1 -- 172.21.15.105:0/922761459 shutdown_connections 2024-09-21T01:46:11.410 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.408+0000 7f06f9c07640 1 -- 172.21.15.105:0/922761459 wait complete. 2024-09-21T01:46:11.425 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705684 got 38654705682 for osd.0 2024-09-21T01:46:11.436 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.435+0000 7f8ca31e4640 1 -- 172.21.15.105:0/109499419 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7f8c9c06a820 con 0x7f8c9c0721b0 2024-09-21T01:46:11.436 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.435+0000 7f8c997fa640 1 -- 172.21.15.105:0/109499419 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f8c8803dc60 con 0x7f8c9c0721b0 2024-09-21T01:46:11.437 INFO:teuthology.orchestra.run.smithi105.stdout:38654705681 2024-09-21T01:46:11.439 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.437+0000 7f8ca31e4640 1 -- 172.21.15.105:0/109499419 >> v1:172.21.15.105:6816/3933313069 conn(0x7f8c800475f0 legacy=0x7f8c80049b00 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:46:11.439 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.437+0000 7f8ca31e4640 1 -- 172.21.15.105:0/109499419 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8c9c0721b0 msgr2=0x7f8c9c0975e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:11.439 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.437+0000 7f8ca31e4640 1 --2- 172.21.15.105:0/109499419 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8c9c0721b0 0x7f8c9c0975e0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f8c88005fd0 tx=0x7f8c88009d30 comp rx=0 tx=0).stop 2024-09-21T01:46:11.439 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.437+0000 7f8ca31e4640 1 -- 172.21.15.105:0/109499419 shutdown_connections 2024-09-21T01:46:11.439 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.437+0000 7f8ca31e4640 1 --2- 172.21.15.105:0/109499419 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8c9c0985f0 0x7f8c9c097b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.439 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.437+0000 7f8ca31e4640 1 --2- 172.21.15.105:0/109499419 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8c9c0721b0 0x7f8c9c0975e0 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.439 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.437+0000 7f8ca31e4640 1 --2- 172.21.15.105:0/109499419 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8c9c071860 0x7f8c9c0970a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:11.439 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.437+0000 7f8ca31e4640 1 -- 172.21.15.105:0/109499419 >> 172.21.15.105:0/109499419 conn(0x7f8c9c09f000 msgr2=0x7f8c9c126d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:11.439 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.437+0000 7f8ca31e4640 1 -- 172.21.15.105:0/109499419 shutdown_connections 2024-09-21T01:46:11.440 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:11.437+0000 7f8ca31e4640 1 -- 172.21.15.105:0/109499419 wait complete. 2024-09-21T01:46:11.450 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705681 for osd.3 2024-09-21T01:46:12.302 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T01:46:12.370 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T01:46:12.419 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.417+0000 7feebbbe2640 1 -- 172.21.15.105:0/2433363501 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7feeb40716c0 msgr2=0x7feeb412fb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:12.420 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.417+0000 7feebbbe2640 1 --2- 172.21.15.105:0/2433363501 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7feeb40716c0 0x7feeb412fb00 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7feeb0009900 tx=0x7feeb002a4a0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.420 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.417+0000 7feebbbe2640 1 -- 172.21.15.105:0/2433363501 shutdown_connections 2024-09-21T01:46:12.420 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.417+0000 7feebbbe2640 1 --2- 172.21.15.105:0/2433363501 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feeb412f5d0 0x7feeb4130040 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.420 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.417+0000 7feebbbe2640 1 --2- 172.21.15.105:0/2433363501 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7feeb40716c0 0x7feeb412fb00 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.420 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.417+0000 7feebbbe2640 1 -- 172.21.15.105:0/2433363501 >> 172.21.15.105:0/2433363501 conn(0x7feeb40a1e40 msgr2=0x7feeb40a2250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:12.420 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.417+0000 7feebbbe2640 1 -- 172.21.15.105:0/2433363501 shutdown_connections 2024-09-21T01:46:12.420 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.417+0000 7feebbbe2640 1 -- 172.21.15.105:0/2433363501 wait complete. 2024-09-21T01:46:12.422 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.419+0000 7feebbbe2640 1 Processor -- start 2024-09-21T01:46:12.422 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.419+0000 7feebbbe2640 1 -- start start 2024-09-21T01:46:12.423 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.420+0000 7feebbbe2640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feeb40716c0 0x7feeb41306e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:12.423 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.420+0000 7feebbbe2640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feeb412c0d0 0x7feeb4130c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:12.423 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.420+0000 7feebbbe2640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7feeb412f5d0 0x7feeb4131160 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:12.423 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.420+0000 7feebbbe2640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7feeb4138610 con 0x7feeb412c0d0 2024-09-21T01:46:12.423 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.420+0000 7feebbbe2640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7feeb4138490 con 0x7feeb40716c0 2024-09-21T01:46:12.423 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.420+0000 7feebbbe2640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7feeb4138790 con 0x7feeb412f5d0 2024-09-21T01:46:12.423 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.420+0000 7feeb997e640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feeb40716c0 0x7feeb41306e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:12.423 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.420+0000 7feeb997e640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feeb40716c0 0x7feeb41306e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:59812/0 (socket says 172.21.15.105:59812) 2024-09-21T01:46:12.423 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.420+0000 7feeb997e640 1 -- 172.21.15.105:0/3462311763 learned_addr learned my addr 172.21.15.105:0/3462311763 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:12.423 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.420+0000 7feeb917d640 1 --2- 172.21.15.105:0/3462311763 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feeb412c0d0 0x7feeb4130c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:12.423 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.420+0000 7feeba17f640 1 --2- 172.21.15.105:0/3462311763 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7feeb412f5d0 0x7feeb4131160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:12.423 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.420+0000 7feeb997e640 1 -- 172.21.15.105:0/3462311763 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7feeb412f5d0 msgr2=0x7feeb4131160 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:12.424 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.420+0000 7feeb997e640 1 --2- 172.21.15.105:0/3462311763 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7feeb412f5d0 0x7feeb4131160 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.424 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.420+0000 7feeb997e640 1 -- 172.21.15.105:0/3462311763 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feeb412c0d0 msgr2=0x7feeb4130c20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:12.424 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.420+0000 7feeb997e640 1 --2- 172.21.15.105:0/3462311763 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feeb412c0d0 0x7feeb4130c20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.424 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.420+0000 7feeb997e640 1 -- 172.21.15.105:0/3462311763 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7feeb4137570 con 0x7feeb40716c0 2024-09-21T01:46:12.424 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.420+0000 7feeba17f640 1 --2- 172.21.15.105:0/3462311763 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7feeb412f5d0 0x7feeb4131160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T01:46:12.424 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.421+0000 7feeb997e640 1 --2- 172.21.15.105:0/3462311763 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feeb40716c0 0x7feeb41306e0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7feeb0006010 tx=0x7feeb00038a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:12.424 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.421+0000 7feeaa7fc640 1 -- 172.21.15.105:0/3462311763 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feeb003e030 con 0x7feeb40716c0 2024-09-21T01:46:12.424 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.421+0000 7feeaa7fc640 1 -- 172.21.15.105:0/3462311763 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7feeb0003e50 con 0x7feeb40716c0 2024-09-21T01:46:12.424 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.421+0000 7feebbbe2640 1 -- 172.21.15.105:0/3462311763 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7feeb4133040 con 0x7feeb40716c0 2024-09-21T01:46:12.424 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.421+0000 7feeaa7fc640 1 -- 172.21.15.105:0/3462311763 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feeb0035040 con 0x7feeb40716c0 2024-09-21T01:46:12.424 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.421+0000 7feebbbe2640 1 -- 172.21.15.105:0/3462311763 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7feeb41cbc70 con 0x7feeb40716c0 2024-09-21T01:46:12.425 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.422+0000 7feeaa7fc640 1 -- 172.21.15.105:0/3462311763 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7feeb0041030 con 0x7feeb40716c0 2024-09-21T01:46:12.425 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.422+0000 7feebbbe2640 1 -- 172.21.15.105:0/3462311763 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7feeb412c4e0 con 0x7feeb40716c0 2024-09-21T01:46:12.425 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.423+0000 7feeaa7fc640 1 -- 172.21.15.105:0/3462311763 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7feeb007d6e0 con 0x7feeb40716c0 2024-09-21T01:46:12.425 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.423+0000 7feeb917d640 1 -- 172.21.15.105:0/3462311763 >> v1:172.21.15.105:6816/3933313069 conn(0x7fee980475f0 legacy=0x7fee98049b00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.105:6816/3933313069 2024-09-21T01:46:12.426 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T01:46:12.431 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.430+0000 7feeaa7fc640 1 -- 172.21.15.105:0/3462311763 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7feeb007c870 con 0x7feeb40716c0 2024-09-21T01:46:12.450 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T01:46:12.502 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.500+0000 7f9b70048640 1 -- 172.21.15.105:0/881027227 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9b681243e0 msgr2=0x7f9b680980b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:12.502 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.500+0000 7f9b70048640 1 --2- 172.21.15.105:0/881027227 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9b681243e0 0x7f9b680980b0 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f9b58005fd0 tx=0x7f9b5802a540 comp rx=0 tx=0).stop 2024-09-21T01:46:12.502 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.500+0000 7f9b70048640 1 -- 172.21.15.105:0/881027227 shutdown_connections 2024-09-21T01:46:12.502 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.500+0000 7f9b70048640 1 --2- 172.21.15.105:0/881027227 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9b681243e0 0x7f9b680980b0 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.502 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.500+0000 7f9b70048640 1 --2- 172.21.15.105:0/881027227 >> v2:172.21.15.105:3300/0 conn(0x7f9b68123a90 0x7f9b68123ea0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.502 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.500+0000 7f9b70048640 1 -- 172.21.15.105:0/881027227 >> 172.21.15.105:0/881027227 conn(0x7f9b6809f000 msgr2=0x7f9b6809b4d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:12.502 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.500+0000 7f9b70048640 1 -- 172.21.15.105:0/881027227 shutdown_connections 2024-09-21T01:46:12.502 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.500+0000 7f9b70048640 1 -- 172.21.15.105:0/881027227 wait complete. 2024-09-21T01:46:12.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.502+0000 7f9b70048640 1 Processor -- start 2024-09-21T01:46:12.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.502+0000 7f9b70048640 1 -- start start 2024-09-21T01:46:12.506 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.504+0000 7f9b70048640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9b680985f0 0x7f9b68128c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:12.506 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.504+0000 7f9b70048640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b68123a90 0x7f9b68129190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:12.506 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.504+0000 7f9b70048640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f9b681243e0 0x7f9b681296d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:12.506 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.504+0000 7f9b6d5e3640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b68123a90 0x7f9b68129190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:12.506 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.504+0000 7f9b6d5e3640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b68123a90 0x7f9b68129190 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:50446/0 (socket says 172.21.15.105:50446) 2024-09-21T01:46:12.506 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.504+0000 7f9b6d5e3640 1 -- 172.21.15.105:0/4073910675 learned_addr learned my addr 172.21.15.105:0/4073910675 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:12.506 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.504+0000 7f9b6dde4640 1 --2- 172.21.15.105:0/4073910675 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9b680985f0 0x7f9b68128c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:12.507 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.505+0000 7f9b6e5e5640 1 --2- 172.21.15.105:0/4073910675 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f9b681243e0 0x7f9b681296d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:12.507 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.505+0000 7f9b70048640 1 -- 172.21.15.105:0/4073910675 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f9b68071bf0 con 0x7f9b68123a90 2024-09-21T01:46:12.507 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.505+0000 7f9b70048640 1 -- 172.21.15.105:0/4073910675 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f9b68071a70 con 0x7f9b680985f0 2024-09-21T01:46:12.507 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.505+0000 7f9b70048640 1 -- 172.21.15.105:0/4073910675 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f9b68071d70 con 0x7f9b681243e0 2024-09-21T01:46:12.508 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.506+0000 7f9b6e5e5640 1 -- 172.21.15.105:0/4073910675 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9b680985f0 msgr2=0x7f9b68128c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:12.508 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.506+0000 7f9b6e5e5640 1 --2- 172.21.15.105:0/4073910675 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9b680985f0 0x7f9b68128c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.508 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.506+0000 7f9b6e5e5640 1 -- 172.21.15.105:0/4073910675 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b68123a90 msgr2=0x7f9b68129190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:12.508 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.506+0000 7f9b6e5e5640 1 --2- 172.21.15.105:0/4073910675 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b68123a90 0x7f9b68129190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.508 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.506+0000 7f9b6e5e5640 1 -- 172.21.15.105:0/4073910675 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9b6808c4c0 con 0x7f9b681243e0 2024-09-21T01:46:12.508 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.506+0000 7f9b6e5e5640 1 --2- 172.21.15.105:0/4073910675 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f9b681243e0 0x7f9b681296d0 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f9b6000c730 tx=0x7f9b6000cbf0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:12.508 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.507+0000 7f9b5e7fc640 1 -- 172.21.15.105:0/4073910675 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9b60003b70 con 0x7f9b681243e0 2024-09-21T01:46:12.508 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.507+0000 7f9b5e7fc640 1 -- 172.21.15.105:0/4073910675 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f9b60003d10 con 0x7f9b681243e0 2024-09-21T01:46:12.508 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.507+0000 7f9b5e7fc640 1 -- 172.21.15.105:0/4073910675 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f9b60005330 con 0x7f9b681243e0 2024-09-21T01:46:12.509 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.507+0000 7f9b70048640 1 -- 172.21.15.105:0/4073910675 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9b6808c750 con 0x7f9b681243e0 2024-09-21T01:46:12.509 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.508+0000 7f9b70048640 1 -- 172.21.15.105:0/4073910675 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f9b681c7ca0 con 0x7f9b681243e0 2024-09-21T01:46:12.511 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.510+0000 7f9b5e7fc640 1 -- 172.21.15.105:0/4073910675 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7f9b600054d0 con 0x7f9b681243e0 2024-09-21T01:46:12.511 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.510+0000 7f9b70048640 1 -- 172.21.15.105:0/4073910675 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9b68123ea0 con 0x7f9b681243e0 2024-09-21T01:46:12.513 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.512+0000 7f9b6dde4640 1 -- 172.21.15.105:0/4073910675 >> v1:172.21.15.105:6816/3933313069 conn(0x7f9b4c047640 legacy=0x7f9b4c049b50 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.105:6816/3933313069 2024-09-21T01:46:12.513 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.512+0000 7f9b5e7fc640 1 -- 172.21.15.105:0/4073910675 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7f9b600057c0 con 0x7f9b681243e0 2024-09-21T01:46:12.519 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.518+0000 7f9b5e7fc640 1 -- 172.21.15.105:0/4073910675 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7f9b6808c750 con 0x7f9b681243e0 2024-09-21T01:46:12.554 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.552+0000 7fc4e6092640 1 -- 172.21.15.105:0/3703492506 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc4e012cb60 msgr2=0x7fc4e00965d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:12.554 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.552+0000 7fc4e6092640 1 --2- 172.21.15.105:0/3703492506 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc4e012cb60 0x7fc4e00965d0 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7fc4d4009830 tx=0x7fc4d402a3f0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.554 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.552+0000 7fc4e6092640 1 -- 172.21.15.105:0/3703492506 shutdown_connections 2024-09-21T01:46:12.554 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.552+0000 7fc4e6092640 1 --2- 172.21.15.105:0/3703492506 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc4e012cb60 0x7fc4e00965d0 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.554 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.552+0000 7fc4e6092640 1 --2- 172.21.15.105:0/3703492506 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fc4e012c210 0x7fc4e012c620 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.554 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.552+0000 7fc4e6092640 1 -- 172.21.15.105:0/3703492506 >> 172.21.15.105:0/3703492506 conn(0x7fc4e009e6f0 msgr2=0x7fc4e009abc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:12.554 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.552+0000 7fc4e6092640 1 -- 172.21.15.105:0/3703492506 shutdown_connections 2024-09-21T01:46:12.554 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.553+0000 7fc4e6092640 1 -- 172.21.15.105:0/3703492506 wait complete. 2024-09-21T01:46:12.555 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.554+0000 7fc4e6092640 1 Processor -- start 2024-09-21T01:46:12.555 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.554+0000 7fc4e6092640 1 -- start start 2024-09-21T01:46:12.557 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.555+0000 7fc4e6092640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fc4e0096b10 0x7fc4e01c3510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:12.557 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.555+0000 7fc4e6092640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fc4e012c210 0x7fc4e01c3a50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:12.557 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.555+0000 7fc4e6092640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc4e012cb60 0x7fc4e01c3f90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:12.557 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.555+0000 7fc4e6092640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7fc4e01345f0 con 0x7fc4e012cb60 2024-09-21T01:46:12.557 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.555+0000 7fc4dffff640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc4e012cb60 0x7fc4e01c3f90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:12.557 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.555+0000 7fc4dffff640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc4e012cb60 0x7fc4e01c3f90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:50450/0 (socket says 172.21.15.105:50450) 2024-09-21T01:46:12.557 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.555+0000 7fc4dffff640 1 -- 172.21.15.105:0/329286283 learned_addr learned my addr 172.21.15.105:0/329286283 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:12.557 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.555+0000 7fc4e6092640 1 -- 172.21.15.105:0/329286283 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7fc4e0134470 con 0x7fc4e0096b10 2024-09-21T01:46:12.557 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.555+0000 7fc4e6092640 1 -- 172.21.15.105:0/329286283 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7fc4e0134770 con 0x7fc4e012c210 2024-09-21T01:46:12.558 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.555+0000 7fc4deffd640 1 --2- 172.21.15.105:0/329286283 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fc4e012c210 0x7fc4e01c3a50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:12.558 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.555+0000 7fc4df7fe640 1 --2- 172.21.15.105:0/329286283 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fc4e0096b10 0x7fc4e01c3510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:12.558 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.556+0000 7fc4dffff640 1 -- 172.21.15.105:0/329286283 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fc4e012c210 msgr2=0x7fc4e01c3a50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:12.558 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.556+0000 7fc4dffff640 1 --2- 172.21.15.105:0/329286283 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fc4e012c210 0x7fc4e01c3a50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.558 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.556+0000 7fc4dffff640 1 -- 172.21.15.105:0/329286283 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fc4e0096b10 msgr2=0x7fc4e01c3510 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:12.558 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.556+0000 7fc4dffff640 1 --2- 172.21.15.105:0/329286283 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fc4e0096b10 0x7fc4e01c3510 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.558 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.556+0000 7fc4dffff640 1 -- 172.21.15.105:0/329286283 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc4e0132620 con 0x7fc4e012cb60 2024-09-21T01:46:12.559 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.556+0000 7fc4dffff640 1 --2- 172.21.15.105:0/329286283 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc4e012cb60 0x7fc4e01c3f90 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7fc4d800c730 tx=0x7fc4d800cbf0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:12.559 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.556+0000 7fc4c7fff640 1 -- 172.21.15.105:0/329286283 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc4d8017070 con 0x7fc4e012cb60 2024-09-21T01:46:12.559 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.557+0000 7fc4e6092640 1 -- 172.21.15.105:0/329286283 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc4e0123b30 con 0x7fc4e012cb60 2024-09-21T01:46:12.559 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.557+0000 7fc4c7fff640 1 -- 172.21.15.105:0/329286283 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fc4d8003c60 con 0x7fc4e012cb60 2024-09-21T01:46:12.559 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.557+0000 7fc4c7fff640 1 -- 172.21.15.105:0/329286283 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc4d80052d0 con 0x7fc4e012cb60 2024-09-21T01:46:12.559 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.557+0000 7fc4e6092640 1 -- 172.21.15.105:0/329286283 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fc4e0131500 con 0x7fc4e012cb60 2024-09-21T01:46:12.559 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.558+0000 7fc4e6092640 1 -- 172.21.15.105:0/329286283 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc4e009f1b0 con 0x7fc4e012cb60 2024-09-21T01:46:12.561 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.559+0000 7fc4c7fff640 1 -- 172.21.15.105:0/329286283 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7fc4d801a070 con 0x7fc4e012cb60 2024-09-21T01:46:12.561 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.559+0000 7fc4c7fff640 1 -- 172.21.15.105:0/329286283 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7fc4d805c2b0 con 0x7fc4e012cb60 2024-09-21T01:46:12.561 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.560+0000 7fc4df7fe640 1 -- 172.21.15.105:0/329286283 >> v1:172.21.15.105:6816/3933313069 conn(0x7fc4cc0475f0 legacy=0x7fc4cc049b00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.105:6816/3933313069 2024-09-21T01:46:12.564 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.563+0000 7fc4c7fff640 1 -- 172.21.15.105:0/329286283 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7fc4d803b9e0 con 0x7fc4e012cb60 2024-09-21T01:46:12.586 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.585+0000 7f656dffb640 1 -- 172.21.15.105:0/207840519 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f656800b830 con 0x7f6570132000 2024-09-21T01:46:12.588 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.586+0000 7f65759f0640 1 -- 172.21.15.105:0/207840519 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6570132000 msgr2=0x7f6570134420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:12.588 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.586+0000 7f65759f0640 1 --2- 172.21.15.105:0/207840519 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6570132000 0x7f6570134420 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f656800f0a0 tx=0x7f656802e870 comp rx=0 tx=0).stop 2024-09-21T01:46:12.588 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.586+0000 7f65759f0640 1 -- 172.21.15.105:0/207840519 shutdown_connections 2024-09-21T01:46:12.588 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.586+0000 7f65759f0640 1 --2- 172.21.15.105:0/207840519 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6570132000 0x7f6570134420 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.588 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.586+0000 7f65759f0640 1 --2- 172.21.15.105:0/207840519 >> v2:172.21.15.105:3300/0 conn(0x7f6570098200 0x7f6570131ac0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.588 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.586+0000 7f65759f0640 1 -- 172.21.15.105:0/207840519 >> 172.21.15.105:0/207840519 conn(0x7f657009e6f0 msgr2=0x7f657009abc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:12.588 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.586+0000 7f65759f0640 1 -- 172.21.15.105:0/207840519 shutdown_connections 2024-09-21T01:46:12.588 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.586+0000 7f65759f0640 1 -- 172.21.15.105:0/207840519 wait complete. 2024-09-21T01:46:12.589 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.588+0000 7f65759f0640 1 Processor -- start 2024-09-21T01:46:12.590 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.588+0000 7f65759f0640 1 -- start start 2024-09-21T01:46:12.591 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.590+0000 7f65759f0640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f65700976e0 0x7f6570072860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:12.592 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.590+0000 7f65759f0640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6570098200 0x7f6570072da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:12.592 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.590+0000 7f656effd640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f65700976e0 0x7f6570072860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:12.592 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.590+0000 7f656effd640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f65700976e0 0x7f6570072860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:50466/0 (socket says 172.21.15.105:50466) 2024-09-21T01:46:12.592 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.590+0000 7f656e7fc640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6570098200 0x7f6570072da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:12.592 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.590+0000 7f656e7fc640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6570098200 0x7f6570072da0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:59864/0 (socket says 172.21.15.105:59864) 2024-09-21T01:46:12.592 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.590+0000 7f65759f0640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6570132000 0x7f65700732e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:12.592 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.590+0000 7f656e7fc640 1 -- 172.21.15.105:0/3224035969 learned_addr learned my addr 172.21.15.105:0/3224035969 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:12.592 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.590+0000 7f65759f0640 1 -- 172.21.15.105:0/3224035969 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f6570097230 con 0x7f65700976e0 2024-09-21T01:46:12.592 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.590+0000 7f65759f0640 1 -- 172.21.15.105:0/3224035969 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f65700970b0 con 0x7f6570098200 2024-09-21T01:46:12.592 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.590+0000 7f65759f0640 1 -- 172.21.15.105:0/3224035969 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f65700973b0 con 0x7f6570132000 2024-09-21T01:46:12.594 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.591+0000 7f656f7fe640 1 --2- 172.21.15.105:0/3224035969 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6570132000 0x7f65700732e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:12.594 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.591+0000 7f656effd640 1 -- 172.21.15.105:0/3224035969 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6570132000 msgr2=0x7f65700732e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:12.594 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.591+0000 7f656effd640 1 --2- 172.21.15.105:0/3224035969 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6570132000 0x7f65700732e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.594 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.591+0000 7f656effd640 1 -- 172.21.15.105:0/3224035969 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6570098200 msgr2=0x7f6570072da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:12.594 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.591+0000 7f656effd640 1 --2- 172.21.15.105:0/3224035969 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6570098200 0x7f6570072da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.594 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.591+0000 7f656effd640 1 -- 172.21.15.105:0/3224035969 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6570074160 con 0x7f65700976e0 2024-09-21T01:46:12.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.591+0000 7f656f7fe640 1 --2- 172.21.15.105:0/3224035969 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6570132000 0x7f65700732e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:12.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.591+0000 7f656e7fc640 1 --2- 172.21.15.105:0/3224035969 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6570098200 0x7f6570072da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:12.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.592+0000 7f656effd640 1 --2- 172.21.15.105:0/3224035969 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f65700976e0 0x7f6570072860 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7f6560001b40 tx=0x7f656000a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:12.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.592+0000 7f6557fff640 1 -- 172.21.15.105:0/3224035969 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f65600050d0 con 0x7f65700976e0 2024-09-21T01:46:12.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.592+0000 7f6557fff640 1 -- 172.21.15.105:0/3224035969 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f6560005360 con 0x7f65700976e0 2024-09-21T01:46:12.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.592+0000 7f6557fff640 1 -- 172.21.15.105:0/3224035969 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6560013b40 con 0x7f65700976e0 2024-09-21T01:46:12.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.592+0000 7f65759f0640 1 -- 172.21.15.105:0/3224035969 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6570124190 con 0x7f65700976e0 2024-09-21T01:46:12.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.592+0000 7f65759f0640 1 -- 172.21.15.105:0/3224035969 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f65701c78b0 con 0x7f65700976e0 2024-09-21T01:46:12.595 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.594+0000 7f6557fff640 1 -- 172.21.15.105:0/3224035969 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 6) ==== 55351+0+0 (secure 0 0 0) 0x7f65600035f0 con 0x7f65700976e0 2024-09-21T01:46:12.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.595+0000 7f656e7fc640 1 -- 172.21.15.105:0/3224035969 >> v1:172.21.15.105:6816/3933313069 conn(0x7f655c0476f0 legacy=0x7f655c049c00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.105:6816/3933313069 2024-09-21T01:46:12.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.595+0000 7f6557fff640 1 -- 172.21.15.105:0/3224035969 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(19..19 src has 1..19) ==== 2812+0+0 (secure 0 0 0) 0x7f6560064bb0 con 0x7f65700976e0 2024-09-21T01:46:12.596 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.595+0000 7f6555ffb640 1 -- 172.21.15.105:0/3224035969 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6548005290 con 0x7f65700976e0 2024-09-21T01:46:12.604 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.603+0000 7f6557fff640 1 -- 172.21.15.105:0/3224035969 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7f6560044300 con 0x7f65700976e0 2024-09-21T01:46:12.625 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.624+0000 7feeb917d640 1 -- 172.21.15.105:0/3462311763 >> v1:172.21.15.105:6816/3933313069 conn(0x7fee980475f0 legacy=0x7fee98049b00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.105:6816/3933313069 2024-09-21T01:46:12.715 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.713+0000 7f9b6dde4640 1 -- 172.21.15.105:0/4073910675 >> v1:172.21.15.105:6816/3933313069 conn(0x7f9b4c047640 legacy=0x7f9b4c049b50 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.105:6816/3933313069 2024-09-21T01:46:12.717 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.715+0000 7fc4e6092640 1 -- 172.21.15.105:0/329286283 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7fc4e01317f0 con 0x7fc4e012cb60 2024-09-21T01:46:12.717 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.716+0000 7fc4c7fff640 1 -- 172.21.15.105:0/329286283 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fc4d803bc40 con 0x7fc4e012cb60 2024-09-21T01:46:12.717 INFO:teuthology.orchestra.run.smithi105.stdout:38654705682 2024-09-21T01:46:12.719 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.718+0000 7fc4e6092640 1 -- 172.21.15.105:0/329286283 >> v1:172.21.15.105:6816/3933313069 conn(0x7fc4cc0475f0 legacy=0x7fc4cc049b00 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:46:12.719 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.718+0000 7fc4e6092640 1 -- 172.21.15.105:0/329286283 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc4e012cb60 msgr2=0x7fc4e01c3f90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:12.720 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.718+0000 7fc4e6092640 1 --2- 172.21.15.105:0/329286283 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc4e012cb60 0x7fc4e01c3f90 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7fc4d800c730 tx=0x7fc4d800cbf0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.720 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.718+0000 7fc4e6092640 1 -- 172.21.15.105:0/329286283 shutdown_connections 2024-09-21T01:46:12.720 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.718+0000 7fc4e6092640 1 --2- 172.21.15.105:0/329286283 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc4e012cb60 0x7fc4e01c3f90 unknown :-1 s=CLOSED pgs=34 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.720 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.718+0000 7fc4e6092640 1 --2- 172.21.15.105:0/329286283 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fc4e012c210 0x7fc4e01c3a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.720 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.718+0000 7fc4e6092640 1 --2- 172.21.15.105:0/329286283 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fc4e0096b10 0x7fc4e01c3510 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.720 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.718+0000 7fc4e6092640 1 -- 172.21.15.105:0/329286283 >> 172.21.15.105:0/329286283 conn(0x7fc4e009e6f0 msgr2=0x7fc4e0098ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:12.720 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.718+0000 7fc4e6092640 1 -- 172.21.15.105:0/329286283 shutdown_connections 2024-09-21T01:46:12.720 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.718+0000 7fc4e6092640 1 -- 172.21.15.105:0/329286283 wait complete. 2024-09-21T01:46:12.730 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705684 got 38654705682 for osd.0 2024-09-21T01:46:12.736 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.735+0000 7feebbbe2640 1 -- 172.21.15.105:0/3462311763 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7feeb4128430 con 0x7feeb40716c0 2024-09-21T01:46:12.737 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.735+0000 7feeaa7fc640 1 -- 172.21.15.105:0/3462311763 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7feeb007c3b0 con 0x7feeb40716c0 2024-09-21T01:46:12.737 INFO:teuthology.orchestra.run.smithi105.stdout:38654705681 2024-09-21T01:46:12.740 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.738+0000 7feebbbe2640 1 -- 172.21.15.105:0/3462311763 >> v1:172.21.15.105:6816/3933313069 conn(0x7fee980475f0 legacy=0x7fee98049b00 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:46:12.740 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.738+0000 7feebbbe2640 1 -- 172.21.15.105:0/3462311763 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feeb40716c0 msgr2=0x7feeb41306e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:12.740 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.738+0000 7feebbbe2640 1 --2- 172.21.15.105:0/3462311763 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feeb40716c0 0x7feeb41306e0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7feeb0006010 tx=0x7feeb00038a0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.740 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.738+0000 7feebbbe2640 1 -- 172.21.15.105:0/3462311763 shutdown_connections 2024-09-21T01:46:12.740 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.738+0000 7feebbbe2640 1 --2- 172.21.15.105:0/3462311763 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7feeb412f5d0 0x7feeb4131160 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.740 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.738+0000 7feebbbe2640 1 --2- 172.21.15.105:0/3462311763 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feeb412c0d0 0x7feeb4130c20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.740 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.739+0000 7feebbbe2640 1 --2- 172.21.15.105:0/3462311763 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feeb40716c0 0x7feeb41306e0 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.740 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.739+0000 7feebbbe2640 1 -- 172.21.15.105:0/3462311763 >> 172.21.15.105:0/3462311763 conn(0x7feeb40a1e40 msgr2=0x7feeb409b8d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:12.741 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.739+0000 7feebbbe2640 1 -- 172.21.15.105:0/3462311763 shutdown_connections 2024-09-21T01:46:12.741 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.739+0000 7feebbbe2640 1 -- 172.21.15.105:0/3462311763 wait complete. 2024-09-21T01:46:12.756 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705681 for osd.1 2024-09-21T01:46:12.797 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.796+0000 7f656e7fc640 1 -- 172.21.15.105:0/3224035969 >> v1:172.21.15.105:6816/3933313069 conn(0x7f655c0476f0 legacy=0x7f655c049c00 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v1:172.21.15.105:6816/3933313069 2024-09-21T01:46:12.822 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.821+0000 7f9b70048640 1 -- 172.21.15.105:0/4073910675 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7f9b6808c260 con 0x7f9b681243e0 2024-09-21T01:46:12.823 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.821+0000 7f9b5e7fc640 1 -- 172.21.15.105:0/4073910675 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f9b6005b240 con 0x7f9b681243e0 2024-09-21T01:46:12.823 INFO:teuthology.orchestra.run.smithi105.stdout:38654705681 2024-09-21T01:46:12.825 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.823+0000 7f9b70048640 1 -- 172.21.15.105:0/4073910675 >> v1:172.21.15.105:6816/3933313069 conn(0x7f9b4c047640 legacy=0x7f9b4c049b50 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:46:12.825 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.823+0000 7f9b70048640 1 -- 172.21.15.105:0/4073910675 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f9b681243e0 msgr2=0x7f9b681296d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:12.825 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.823+0000 7f9b70048640 1 --2- 172.21.15.105:0/4073910675 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f9b681243e0 0x7f9b681296d0 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f9b6000c730 tx=0x7f9b6000cbf0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.825 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.823+0000 7f9b70048640 1 -- 172.21.15.105:0/4073910675 shutdown_connections 2024-09-21T01:46:12.825 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.823+0000 7f9b70048640 1 --2- 172.21.15.105:0/4073910675 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f9b681243e0 0x7f9b681296d0 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.825 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.823+0000 7f9b70048640 1 --2- 172.21.15.105:0/4073910675 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f9b68123a90 0x7f9b68129190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.825 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.823+0000 7f9b70048640 1 --2- 172.21.15.105:0/4073910675 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9b680985f0 0x7f9b68128c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.826 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.823+0000 7f9b70048640 1 -- 172.21.15.105:0/4073910675 >> 172.21.15.105:0/4073910675 conn(0x7f9b6809f000 msgr2=0x7f9b68073140 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:12.826 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.823+0000 7f9b70048640 1 -- 172.21.15.105:0/4073910675 shutdown_connections 2024-09-21T01:46:12.826 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.823+0000 7f9b70048640 1 -- 172.21.15.105:0/4073910675 wait complete. 2024-09-21T01:46:12.837 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705681 for osd.2 2024-09-21T01:46:12.868 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.866+0000 7f6555ffb640 1 -- 172.21.15.105:0/3224035969 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7f6548005580 con 0x7f65700976e0 2024-09-21T01:46:12.871 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.867+0000 7f6557fff640 1 -- 172.21.15.105:0/3224035969 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f6560044560 con 0x7f65700976e0 2024-09-21T01:46:12.875 INFO:teuthology.orchestra.run.smithi105.stdout:38654705681 2024-09-21T01:46:12.875 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.868+0000 7f6555ffb640 1 -- 172.21.15.105:0/3224035969 >> v1:172.21.15.105:6816/3933313069 conn(0x7f655c0476f0 legacy=0x7f655c049c00 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:46:12.875 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.868+0000 7f6555ffb640 1 -- 172.21.15.105:0/3224035969 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f65700976e0 msgr2=0x7f6570072860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:12.876 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.868+0000 7f6555ffb640 1 --2- 172.21.15.105:0/3224035969 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f65700976e0 0x7f6570072860 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7f6560001b40 tx=0x7f656000a430 comp rx=0 tx=0).stop 2024-09-21T01:46:12.876 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.868+0000 7f6555ffb640 1 -- 172.21.15.105:0/3224035969 shutdown_connections 2024-09-21T01:46:12.876 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.868+0000 7f6555ffb640 1 --2- 172.21.15.105:0/3224035969 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6570132000 0x7f65700732e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.876 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.868+0000 7f6555ffb640 1 --2- 172.21.15.105:0/3224035969 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6570098200 0x7f6570072da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.876 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.868+0000 7f6555ffb640 1 --2- 172.21.15.105:0/3224035969 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f65700976e0 0x7f6570072860 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:12.876 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.868+0000 7f6555ffb640 1 -- 172.21.15.105:0/3224035969 >> 172.21.15.105:0/3224035969 conn(0x7f657009e6f0 msgr2=0x7f657009aad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:12.876 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.869+0000 7f6555ffb640 1 -- 172.21.15.105:0/3224035969 shutdown_connections 2024-09-21T01:46:12.876 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:12.869+0000 7f6555ffb640 1 -- 172.21.15.105:0/3224035969 wait complete. 2024-09-21T01:46:12.881 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705681 for osd.3 2024-09-21T01:46:13.731 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T01:46:13.757 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T01:46:13.837 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T01:46:13.846 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.845+0000 7f130608b640 1 -- 172.21.15.105:0/2254596627 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1300072380 msgr2=0x7f1300131970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:13.846 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.845+0000 7f130608b640 1 --2- 172.21.15.105:0/2254596627 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1300072380 0x7f1300131970 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f12f4009900 tx=0x7f12f402a610 comp rx=0 tx=0).stop 2024-09-21T01:46:13.846 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.845+0000 7f130608b640 1 -- 172.21.15.105:0/2254596627 shutdown_connections 2024-09-21T01:46:13.846 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.845+0000 7f130608b640 1 --2- 172.21.15.105:0/2254596627 >> v2:172.21.15.105:3300/0 conn(0x7f1300131eb0 0x7f13001342d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:13.846 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.845+0000 7f130608b640 1 --2- 172.21.15.105:0/2254596627 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1300072380 0x7f1300131970 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:13.846 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.845+0000 7f130608b640 1 -- 172.21.15.105:0/2254596627 >> 172.21.15.105:0/2254596627 conn(0x7f130009f000 msgr2=0x7f130009b4d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:13.846 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.845+0000 7f130608b640 1 -- 172.21.15.105:0/2254596627 shutdown_connections 2024-09-21T01:46:13.846 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.845+0000 7f130608b640 1 -- 172.21.15.105:0/2254596627 wait complete. 2024-09-21T01:46:13.847 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.847+0000 7f130608b640 1 Processor -- start 2024-09-21T01:46:13.847 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.847+0000 7f130608b640 1 -- start start 2024-09-21T01:46:13.848 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.847+0000 7f130608b640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f1300071860 0x7f1300098480 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:13.848 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.847+0000 7f130608b640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1300072380 0x7f13000989c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:13.848 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.847+0000 7f130608b640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1300131eb0 0x7f130006e9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:13.849 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.848+0000 7f130608b640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f1300135420 con 0x7f1300072380 2024-09-21T01:46:13.849 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.848+0000 7f130608b640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f13001352a0 con 0x7f1300071860 2024-09-21T01:46:13.849 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.848+0000 7f130608b640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f13001355a0 con 0x7f1300131eb0 2024-09-21T01:46:13.849 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.848+0000 7f12feffd640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1300072380 0x7f13000989c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:13.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.848+0000 7f12fffff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1300131eb0 0x7f130006e9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:13.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.848+0000 7f12feffd640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1300072380 0x7f13000989c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:50556/0 (socket says 172.21.15.105:50556) 2024-09-21T01:46:13.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.848+0000 7f12feffd640 1 -- 172.21.15.105:0/2642560704 learned_addr learned my addr 172.21.15.105:0/2642560704 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:13.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.848+0000 7f12ff7fe640 1 --2- 172.21.15.105:0/2642560704 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f1300071860 0x7f1300098480 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:13.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.848+0000 7f12fffff640 1 -- 172.21.15.105:0/2642560704 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f1300071860 msgr2=0x7f1300098480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:13.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.848+0000 7f12fffff640 1 --2- 172.21.15.105:0/2642560704 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f1300071860 0x7f1300098480 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:13.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.848+0000 7f12fffff640 1 -- 172.21.15.105:0/2642560704 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1300072380 msgr2=0x7f13000989c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:13.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.848+0000 7f12fffff640 1 --2- 172.21.15.105:0/2642560704 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1300072380 0x7f13000989c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:13.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.848+0000 7f12fffff640 1 -- 172.21.15.105:0/2642560704 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f130012d9c0 con 0x7f1300131eb0 2024-09-21T01:46:13.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.848+0000 7f12ff7fe640 1 --2- 172.21.15.105:0/2642560704 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f1300071860 0x7f1300098480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:13.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.848+0000 7f12fffff640 1 --2- 172.21.15.105:0/2642560704 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1300131eb0 0x7f130006e9e0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f12f8005fd0 tx=0x7f12f800ee40 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:13.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.849+0000 7f12feffd640 1 --2- 172.21.15.105:0/2642560704 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1300072380 0x7f13000989c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:13.850 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.849+0000 7f12e7fff640 1 -- 172.21.15.105:0/2642560704 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f12f800cd80 con 0x7f1300131eb0 2024-09-21T01:46:13.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.849+0000 7f130608b640 1 -- 172.21.15.105:0/2642560704 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1300129050 con 0x7f1300131eb0 2024-09-21T01:46:13.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.849+0000 7f12e7fff640 1 -- 172.21.15.105:0/2642560704 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f12f8004c10 con 0x7f1300131eb0 2024-09-21T01:46:13.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.849+0000 7f12e7fff640 1 -- 172.21.15.105:0/2642560704 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f12f8005020 con 0x7f1300131eb0 2024-09-21T01:46:13.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.849+0000 7f130608b640 1 -- 172.21.15.105:0/2642560704 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f130006f100 con 0x7f1300131eb0 2024-09-21T01:46:13.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.850+0000 7f130608b640 1 -- 172.21.15.105:0/2642560704 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f1300126b40 con 0x7f1300131eb0 2024-09-21T01:46:13.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.850+0000 7f12e7fff640 1 -- 172.21.15.105:0/2642560704 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 7) ==== 59167+0+0 (secure 0 0 0) 0x7f12f80051c0 con 0x7f1300131eb0 2024-09-21T01:46:13.851 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.851+0000 7f12e7fff640 1 -- 172.21.15.105:0/2642560704 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7f12f80637a0 con 0x7f1300131eb0 2024-09-21T01:46:13.855 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.855+0000 7f12e7fff640 1 -- 172.21.15.105:0/2642560704 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7f12f8062360 con 0x7f1300131eb0 2024-09-21T01:46:13.882 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T01:46:13.889 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.889+0000 7f3d6c389640 1 -- 172.21.15.105:0/3381578895 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3d6412c000 msgr2=0x7f3d64135130 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:13.890 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.889+0000 7f3d6c389640 1 --2- 172.21.15.105:0/3381578895 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3d6412c000 0x7f3d64135130 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f3d5c009be0 tx=0x7f3d5c02a800 comp rx=0 tx=0).stop 2024-09-21T01:46:13.890 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.889+0000 7f3d6c389640 1 -- 172.21.15.105:0/3381578895 shutdown_connections 2024-09-21T01:46:13.890 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.889+0000 7f3d6c389640 1 --2- 172.21.15.105:0/3381578895 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3d6412c000 0x7f3d64135130 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:13.890 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.889+0000 7f3d6c389640 1 --2- 172.21.15.105:0/3381578895 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3d6412bbf0 0x7f3d64132be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:13.890 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.889+0000 7f3d6c389640 1 -- 172.21.15.105:0/3381578895 >> 172.21.15.105:0/3381578895 conn(0x7f3d640a0160 msgr2=0x7f3d6409c630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:13.890 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.889+0000 7f3d6c389640 1 -- 172.21.15.105:0/3381578895 shutdown_connections 2024-09-21T01:46:13.890 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.889+0000 7f3d6c389640 1 -- 172.21.15.105:0/3381578895 wait complete. 2024-09-21T01:46:13.890 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.890+0000 7f3d6c389640 1 Processor -- start 2024-09-21T01:46:13.891 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.890+0000 7f3d6c389640 1 -- start start 2024-09-21T01:46:13.891 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.891+0000 7f3d6c389640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3d64073dd0 0x7f3d64130a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:13.891 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.891+0000 7f3d6c389640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3d6412bbf0 0x7f3d64130f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:13.891 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.891+0000 7f3d6c389640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3d6412c000 0x7f3d641314a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:13.891 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.891+0000 7f3d6a125640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3d64073dd0 0x7f3d64130a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:13.892 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.891+0000 7f3d6a125640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3d64073dd0 0x7f3d64130a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:50560/0 (socket says 172.21.15.105:50560) 2024-09-21T01:46:13.892 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.891+0000 7f3d6c389640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f3d64136860 con 0x7f3d64073dd0 2024-09-21T01:46:13.892 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.891+0000 7f3d6a125640 1 -- 172.21.15.105:0/992090525 learned_addr learned my addr 172.21.15.105:0/992090525 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:13.892 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.891+0000 7f3d6c389640 1 -- 172.21.15.105:0/992090525 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f3d641366e0 con 0x7f3d6412bbf0 2024-09-21T01:46:13.892 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.891+0000 7f3d6c389640 1 -- 172.21.15.105:0/992090525 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f3d641369e0 con 0x7f3d6412c000 2024-09-21T01:46:13.892 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.891+0000 7f3d6a926640 1 --2- 172.21.15.105:0/992090525 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3d6412c000 0x7f3d641314a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:13.892 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.891+0000 7f3d69924640 1 --2- 172.21.15.105:0/992090525 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3d6412bbf0 0x7f3d64130f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:13.893 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.892+0000 7f3d6a926640 1 -- 172.21.15.105:0/992090525 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3d6412bbf0 msgr2=0x7f3d64130f60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:13.893 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.892+0000 7f3d6a926640 1 --2- 172.21.15.105:0/992090525 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3d6412bbf0 0x7f3d64130f60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:13.893 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.892+0000 7f3d6a926640 1 -- 172.21.15.105:0/992090525 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3d64073dd0 msgr2=0x7f3d64130a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:13.893 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.892+0000 7f3d6a926640 1 --2- 172.21.15.105:0/992090525 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3d64073dd0 0x7f3d64130a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:13.893 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.892+0000 7f3d6a926640 1 -- 172.21.15.105:0/992090525 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3d64127e30 con 0x7f3d6412c000 2024-09-21T01:46:13.894 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.892+0000 7f3d6a125640 1 --2- 172.21.15.105:0/992090525 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3d64073dd0 0x7f3d64130a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:13.894 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.892+0000 7f3d69924640 1 --2- 172.21.15.105:0/992090525 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3d6412bbf0 0x7f3d64130f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T01:46:13.894 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.892+0000 7f3d6a926640 1 --2- 172.21.15.105:0/992090525 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3d6412c000 0x7f3d641314a0 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f3d6409c570 tx=0x7f3d5c009480 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:13.894 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.892+0000 7f3d5affd640 1 -- 172.21.15.105:0/992090525 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3d5c037040 con 0x7f3d6412c000 2024-09-21T01:46:13.894 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.892+0000 7f3d6c389640 1 -- 172.21.15.105:0/992090525 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3d64134a20 con 0x7f3d6412c000 2024-09-21T01:46:13.894 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.892+0000 7f3d6c389640 1 -- 172.21.15.105:0/992090525 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f3d64133240 con 0x7f3d6412c000 2024-09-21T01:46:13.894 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.892+0000 7f3d5affd640 1 -- 172.21.15.105:0/992090525 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f3d5c004270 con 0x7f3d6412c000 2024-09-21T01:46:13.894 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.893+0000 7f3d5affd640 1 -- 172.21.15.105:0/992090525 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3d5c033e60 con 0x7f3d6412c000 2024-09-21T01:46:13.895 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.894+0000 7f3d5affd640 1 -- 172.21.15.105:0/992090525 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 7) ==== 59167+0+0 (secure 0 0 0) 0x7f3d5c004620 con 0x7f3d6412c000 2024-09-21T01:46:13.895 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.894+0000 7f3d6c389640 1 -- 172.21.15.105:0/992090525 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f3d4c005290 con 0x7f3d6412c000 2024-09-21T01:46:13.895 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.894+0000 7f3d5affd640 1 -- 172.21.15.105:0/992090525 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7f3d5c087520 con 0x7f3d6412c000 2024-09-21T01:46:13.898 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.898+0000 7f3d5affd640 1 -- 172.21.15.105:0/992090525 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7f3d5c02f070 con 0x7f3d6412c000 2024-09-21T01:46:13.976 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.975+0000 7f279ebf0640 1 -- 172.21.15.105:0/2137159276 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27980717b0 msgr2=0x7f279812c570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:13.976 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.975+0000 7f279ebf0640 1 --2- 172.21.15.105:0/2137159276 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27980717b0 0x7f279812c570 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f278c009900 tx=0x7f278c02a440 comp rx=0 tx=0).stop 2024-09-21T01:46:13.976 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.975+0000 7f279ebf0640 1 -- 172.21.15.105:0/2137159276 shutdown_connections 2024-09-21T01:46:13.976 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.975+0000 7f279ebf0640 1 --2- 172.21.15.105:0/2137159276 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f279812fd90 0x7f2798137a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:13.976 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.975+0000 7f279ebf0640 1 --2- 172.21.15.105:0/2137159276 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27980717b0 0x7f279812c570 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:13.976 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.975+0000 7f279ebf0640 1 -- 172.21.15.105:0/2137159276 >> 172.21.15.105:0/2137159276 conn(0x7f27980a2180 msgr2=0x7f27980a2590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:13.976 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.975+0000 7f279ebf0640 1 -- 172.21.15.105:0/2137159276 shutdown_connections 2024-09-21T01:46:13.976 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.975+0000 7f279ebf0640 1 -- 172.21.15.105:0/2137159276 wait complete. 2024-09-21T01:46:13.977 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.977+0000 7f279ebf0640 1 Processor -- start 2024-09-21T01:46:13.977 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.977+0000 7f279ebf0640 1 -- start start 2024-09-21T01:46:13.979 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.978+0000 7f279ebf0640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27980717b0 0x7f279809b4b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:13.979 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.978+0000 7f279ebf0640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f279812cab0 0x7f279809b9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:13.979 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.978+0000 7f2797fff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f279812cab0 0x7f279809b9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:13.979 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.978+0000 7f2797fff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f279812cab0 0x7f279809b9f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:59970/0 (socket says 172.21.15.105:59970) 2024-09-21T01:46:13.979 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.978+0000 7f2797fff640 1 -- 172.21.15.105:0/2270465703 learned_addr learned my addr 172.21.15.105:0/2270465703 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:13.979 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.978+0000 7f279ebf0640 1 --2- 172.21.15.105:0/2270465703 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f279812fd90 0x7f279809bf30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:13.979 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.978+0000 7f279c98c640 1 --2- 172.21.15.105:0/2270465703 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27980717b0 0x7f279809b4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:13.980 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.978+0000 7f279ebf0640 1 -- 172.21.15.105:0/2270465703 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f2798138720 con 0x7f27980717b0 2024-09-21T01:46:13.980 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.978+0000 7f279ebf0640 1 -- 172.21.15.105:0/2270465703 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f27981385a0 con 0x7f279812cab0 2024-09-21T01:46:13.980 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.978+0000 7f279ebf0640 1 -- 172.21.15.105:0/2270465703 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f27981388a0 con 0x7f279812fd90 2024-09-21T01:46:13.980 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.978+0000 7f279d18d640 1 --2- 172.21.15.105:0/2270465703 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f279812fd90 0x7f279809bf30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:13.981 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.979+0000 7f2797fff640 1 -- 172.21.15.105:0/2270465703 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f279812fd90 msgr2=0x7f279809bf30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:13.981 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.979+0000 7f2797fff640 1 --2- 172.21.15.105:0/2270465703 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f279812fd90 0x7f279809bf30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:13.981 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.979+0000 7f2797fff640 1 -- 172.21.15.105:0/2270465703 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27980717b0 msgr2=0x7f279809b4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:13.981 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.979+0000 7f2797fff640 1 --2- 172.21.15.105:0/2270465703 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27980717b0 0x7f279809b4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:13.981 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.979+0000 7f2797fff640 1 -- 172.21.15.105:0/2270465703 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f2798134470 con 0x7f279812cab0 2024-09-21T01:46:13.981 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.979+0000 7f279c98c640 1 --2- 172.21.15.105:0/2270465703 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27980717b0 0x7f279809b4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:13.981 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.979+0000 7f279d18d640 1 --2- 172.21.15.105:0/2270465703 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f279812fd90 0x7f279809bf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:13.981 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.979+0000 7f2797fff640 1 --2- 172.21.15.105:0/2270465703 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f279812cab0 0x7f279809b9f0 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f278400c780 tx=0x7f278400cc40 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:13.981 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.980+0000 7f27957fa640 1 -- 172.21.15.105:0/2270465703 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2784017070 con 0x7f279812cab0 2024-09-21T01:46:13.981 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.980+0000 7f279ebf0640 1 -- 172.21.15.105:0/2270465703 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2798133bd0 con 0x7f279812cab0 2024-09-21T01:46:13.981 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.980+0000 7f279ebf0640 1 -- 172.21.15.105:0/2270465703 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f2798131b70 con 0x7f279812cab0 2024-09-21T01:46:13.981 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.980+0000 7f27957fa640 1 -- 172.21.15.105:0/2270465703 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2784005380 con 0x7f279812cab0 2024-09-21T01:46:13.982 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.980+0000 7f27957fa640 1 -- 172.21.15.105:0/2270465703 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f278400a8a0 con 0x7f279812cab0 2024-09-21T01:46:13.982 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.981+0000 7f279ebf0640 1 -- 172.21.15.105:0/2270465703 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f2778005290 con 0x7f279812cab0 2024-09-21T01:46:13.982 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.981+0000 7f27957fa640 1 -- 172.21.15.105:0/2270465703 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 7) ==== 59167+0+0 (secure 0 0 0) 0x7f278400aa40 con 0x7f279812cab0 2024-09-21T01:46:13.982 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.982+0000 7f27957fa640 1 -- 172.21.15.105:0/2270465703 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7f2784061dd0 con 0x7f279812cab0 2024-09-21T01:46:13.989 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:13.989+0000 7f27957fa640 1 -- 172.21.15.105:0/2270465703 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7f278400ad30 con 0x7f279812cab0 2024-09-21T01:46:14.005 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.004+0000 7fdde2026640 1 -- 172.21.15.105:0/4174942607 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fdddc130900 msgr2=0x7fdddc132d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:14.005 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.004+0000 7fdde2026640 1 --2- 172.21.15.105:0/4174942607 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fdddc130900 0x7fdddc132d20 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7fddd0009860 tx=0x7fddd002a360 comp rx=0 tx=0).stop 2024-09-21T01:46:14.005 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.005+0000 7fdde2026640 1 -- 172.21.15.105:0/4174942607 shutdown_connections 2024-09-21T01:46:14.005 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.005+0000 7fdde2026640 1 --2- 172.21.15.105:0/4174942607 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fdddc133260 0x7fdddc135680 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:14.005 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.005+0000 7fdde2026640 1 --2- 172.21.15.105:0/4174942607 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fdddc130900 0x7fdddc132d20 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:14.005 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.005+0000 7fdde2026640 1 -- 172.21.15.105:0/4174942607 >> 172.21.15.105:0/4174942607 conn(0x7fdddc0a00f0 msgr2=0x7fdddc09c5c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:14.005 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.005+0000 7fdde2026640 1 -- 172.21.15.105:0/4174942607 shutdown_connections 2024-09-21T01:46:14.005 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.005+0000 7fdde2026640 1 -- 172.21.15.105:0/4174942607 wait complete. 2024-09-21T01:46:14.006 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.006+0000 7fdde2026640 1 Processor -- start 2024-09-21T01:46:14.006 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.006+0000 7fdde2026640 1 -- start start 2024-09-21T01:46:14.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.007+0000 7fdde2026640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fdddc12a5e0 0x7fdddc12b370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:14.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.007+0000 7fdde2026640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fdddc130900 0x7fdddc12b8b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:14.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.007+0000 7fdde2026640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdddc133260 0x7fdddc12bdf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:14.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.007+0000 7fdde2026640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7fdddc136770 con 0x7fdddc133260 2024-09-21T01:46:14.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.007+0000 7fdddaffd640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fdddc130900 0x7fdddc12b8b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:14.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.007+0000 7fdde2026640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7fdddc1365f0 con 0x7fdddc130900 2024-09-21T01:46:14.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.007+0000 7fdddb7fe640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fdddc12a5e0 0x7fdddc12b370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:14.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.007+0000 7fdddaffd640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fdddc130900 0x7fdddc12b8b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:59988/0 (socket says 172.21.15.105:59988) 2024-09-21T01:46:14.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.007+0000 7fdddaffd640 1 -- 172.21.15.105:0/2950486097 learned_addr learned my addr 172.21.15.105:0/2950486097 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:14.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.007+0000 7fdddbfff640 1 --2- 172.21.15.105:0/2950486097 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdddc133260 0x7fdddc12bdf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:14.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.007+0000 7fdde2026640 1 -- 172.21.15.105:0/2950486097 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7fdddc1368f0 con 0x7fdddc12a5e0 2024-09-21T01:46:14.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.008+0000 7fdddb7fe640 1 -- 172.21.15.105:0/2950486097 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fdddc130900 msgr2=0x7fdddc12b8b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:14.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.008+0000 7fdddb7fe640 1 --2- 172.21.15.105:0/2950486097 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fdddc130900 0x7fdddc12b8b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:14.010 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.008+0000 7fdddb7fe640 1 -- 172.21.15.105:0/2950486097 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdddc133260 msgr2=0x7fdddc12bdf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:14.010 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.008+0000 7fdddb7fe640 1 --2- 172.21.15.105:0/2950486097 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdddc133260 0x7fdddc12bdf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:14.010 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.008+0000 7fdddb7fe640 1 -- 172.21.15.105:0/2950486097 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fdddc12c4a0 con 0x7fdddc12a5e0 2024-09-21T01:46:14.010 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.008+0000 7fdddbfff640 1 --2- 172.21.15.105:0/2950486097 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdddc133260 0x7fdddc12bdf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:14.010 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.008+0000 7fdddb7fe640 1 --2- 172.21.15.105:0/2950486097 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fdddc12a5e0 0x7fdddc12b370 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7fddcc00c840 tx=0x7fddcc00cd00 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:14.010 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.008+0000 7fddc3fff640 1 -- 172.21.15.105:0/2950486097 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fddcc017070 con 0x7fdddc12a5e0 2024-09-21T01:46:14.010 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.008+0000 7fdde2026640 1 -- 172.21.15.105:0/2950486097 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fdddc1273d0 con 0x7fdddc12a5e0 2024-09-21T01:46:14.010 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.008+0000 7fddc3fff640 1 -- 172.21.15.105:0/2950486097 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fddcc003da0 con 0x7fdddc12a5e0 2024-09-21T01:46:14.010 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.008+0000 7fddc3fff640 1 -- 172.21.15.105:0/2950486097 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fddcc005330 con 0x7fdddc12a5e0 2024-09-21T01:46:14.010 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.008+0000 7fdde2026640 1 -- 172.21.15.105:0/2950486097 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fdddc1c3bc0 con 0x7fdddc12a5e0 2024-09-21T01:46:14.011 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.009+0000 7fddc3fff640 1 -- 172.21.15.105:0/2950486097 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 7) ==== 59167+0+0 (secure 0 0 0) 0x7fddcc01a070 con 0x7fdddc12a5e0 2024-09-21T01:46:14.011 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.009+0000 7fddc3fff640 1 -- 172.21.15.105:0/2950486097 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7fddcc062350 con 0x7fdddc12a5e0 2024-09-21T01:46:14.011 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.009+0000 7fdde2026640 1 -- 172.21.15.105:0/2950486097 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fdddc12a9f0 con 0x7fdddc12a5e0 2024-09-21T01:46:14.014 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.014+0000 7fddc3fff640 1 -- 172.21.15.105:0/2950486097 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+148921 (secure 0 0 0) 0x7fddcc0611a0 con 0x7fdddc12a5e0 2024-09-21T01:46:14.028 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.028+0000 7f130608b640 1 -- 172.21.15.105:0/2642560704 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7f130012df00 con 0x7f1300131eb0 2024-09-21T01:46:14.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.029+0000 7f12e7fff640 1 -- 172.21.15.105:0/2642560704 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f12f80626e0 con 0x7f1300131eb0 2024-09-21T01:46:14.031 INFO:teuthology.orchestra.run.smithi105.stdout:38654705682 2024-09-21T01:46:14.033 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.032+0000 7f130608b640 1 -- 172.21.15.105:0/2642560704 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1300131eb0 msgr2=0x7f130006e9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:14.033 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.032+0000 7f130608b640 1 --2- 172.21.15.105:0/2642560704 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1300131eb0 0x7f130006e9e0 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f12f8005fd0 tx=0x7f12f800ee40 comp rx=0 tx=0).stop 2024-09-21T01:46:14.033 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.032+0000 7f130608b640 1 -- 172.21.15.105:0/2642560704 shutdown_connections 2024-09-21T01:46:14.033 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.032+0000 7f130608b640 1 --2- 172.21.15.105:0/2642560704 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f1300131eb0 0x7f130006e9e0 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:14.033 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.032+0000 7f130608b640 1 --2- 172.21.15.105:0/2642560704 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f1300072380 0x7f13000989c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:14.033 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.032+0000 7f130608b640 1 --2- 172.21.15.105:0/2642560704 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f1300071860 0x7f1300098480 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:14.033 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.032+0000 7f130608b640 1 -- 172.21.15.105:0/2642560704 >> 172.21.15.105:0/2642560704 conn(0x7f130009f000 msgr2=0x7f130009afa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:14.033 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.032+0000 7f130608b640 1 -- 172.21.15.105:0/2642560704 shutdown_connections 2024-09-21T01:46:14.033 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.032+0000 7f130608b640 1 -- 172.21.15.105:0/2642560704 wait complete. 2024-09-21T01:46:14.047 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705684 got 38654705682 for osd.0 2024-09-21T01:46:14.078 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.078+0000 7f3d6c389640 1 -- 172.21.15.105:0/992090525 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7f3d4c005580 con 0x7f3d6412c000 2024-09-21T01:46:14.079 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.078+0000 7f3d5affd640 1 -- 172.21.15.105:0/992090525 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f3d5c04a0c0 con 0x7f3d6412c000 2024-09-21T01:46:14.079 INFO:teuthology.orchestra.run.smithi105.stdout:38654705681 2024-09-21T01:46:14.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.080+0000 7f3d6c389640 1 -- 172.21.15.105:0/992090525 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3d6412c000 msgr2=0x7f3d641314a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:14.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.080+0000 7f3d6c389640 1 --2- 172.21.15.105:0/992090525 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3d6412c000 0x7f3d641314a0 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f3d6409c570 tx=0x7f3d5c009480 comp rx=0 tx=0).stop 2024-09-21T01:46:14.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.080+0000 7f3d6c389640 1 -- 172.21.15.105:0/992090525 shutdown_connections 2024-09-21T01:46:14.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.080+0000 7f3d6c389640 1 --2- 172.21.15.105:0/992090525 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3d6412c000 0x7f3d641314a0 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:14.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.080+0000 7f3d6c389640 1 --2- 172.21.15.105:0/992090525 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3d6412bbf0 0x7f3d64130f60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:14.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.080+0000 7f3d6c389640 1 --2- 172.21.15.105:0/992090525 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3d64073dd0 0x7f3d64130a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:14.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.080+0000 7f3d6c389640 1 -- 172.21.15.105:0/992090525 >> 172.21.15.105:0/992090525 conn(0x7f3d640a0160 msgr2=0x7f3d6412ddd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:14.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.080+0000 7f3d6c389640 1 -- 172.21.15.105:0/992090525 shutdown_connections 2024-09-21T01:46:14.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.080+0000 7f3d6c389640 1 -- 172.21.15.105:0/992090525 wait complete. 2024-09-21T01:46:14.091 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705681 for osd.1 2024-09-21T01:46:14.158 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.158+0000 7fdde2026640 1 -- 172.21.15.105:0/2950486097 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7fdddc06a690 con 0x7fdddc12a5e0 2024-09-21T01:46:14.158 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.158+0000 7fddc3fff640 1 -- 172.21.15.105:0/2950486097 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fddcc03b280 con 0x7fdddc12a5e0 2024-09-21T01:46:14.159 INFO:teuthology.orchestra.run.smithi105.stdout:38654705681 2024-09-21T01:46:14.159 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.159+0000 7f279ebf0640 1 -- 172.21.15.105:0/2270465703 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7f2778005580 con 0x7f279812cab0 2024-09-21T01:46:14.159 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.159+0000 7f27957fa640 1 -- 172.21.15.105:0/2270465703 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f2784063020 con 0x7f279812cab0 2024-09-21T01:46:14.159 INFO:teuthology.orchestra.run.smithi105.stdout:38654705681 2024-09-21T01:46:14.161 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.160+0000 7fdde2026640 1 -- 172.21.15.105:0/2950486097 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fdddc12a5e0 msgr2=0x7fdddc12b370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:14.161 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.160+0000 7fdde2026640 1 --2- 172.21.15.105:0/2950486097 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fdddc12a5e0 0x7fdddc12b370 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7fddcc00c840 tx=0x7fddcc00cd00 comp rx=0 tx=0).stop 2024-09-21T01:46:14.161 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.160+0000 7fdde2026640 1 -- 172.21.15.105:0/2950486097 shutdown_connections 2024-09-21T01:46:14.161 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.160+0000 7fdde2026640 1 --2- 172.21.15.105:0/2950486097 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fdddc133260 0x7fdddc12bdf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:14.161 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.160+0000 7fdde2026640 1 --2- 172.21.15.105:0/2950486097 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fdddc130900 0x7fdddc12b8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:14.161 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.160+0000 7fdde2026640 1 --2- 172.21.15.105:0/2950486097 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fdddc12a5e0 0x7fdddc12b370 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:14.161 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.160+0000 7fdde2026640 1 -- 172.21.15.105:0/2950486097 >> 172.21.15.105:0/2950486097 conn(0x7fdddc0a00f0 msgr2=0x7fdddc127f10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:14.161 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.160+0000 7fdde2026640 1 -- 172.21.15.105:0/2950486097 shutdown_connections 2024-09-21T01:46:14.161 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.160+0000 7fdde2026640 1 -- 172.21.15.105:0/2950486097 wait complete. 2024-09-21T01:46:14.162 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.161+0000 7f279ebf0640 1 -- 172.21.15.105:0/2270465703 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f279812cab0 msgr2=0x7f279809b9f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:14.162 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.161+0000 7f279ebf0640 1 --2- 172.21.15.105:0/2270465703 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f279812cab0 0x7f279809b9f0 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f278400c780 tx=0x7f278400cc40 comp rx=0 tx=0).stop 2024-09-21T01:46:14.162 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.161+0000 7f279ebf0640 1 -- 172.21.15.105:0/2270465703 shutdown_connections 2024-09-21T01:46:14.162 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.161+0000 7f279ebf0640 1 --2- 172.21.15.105:0/2270465703 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f279812fd90 0x7f279809bf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:14.162 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.161+0000 7f279ebf0640 1 --2- 172.21.15.105:0/2270465703 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f279812cab0 0x7f279809b9f0 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:14.162 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.161+0000 7f279ebf0640 1 --2- 172.21.15.105:0/2270465703 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27980717b0 0x7f279809b4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:14.162 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.161+0000 7f279ebf0640 1 -- 172.21.15.105:0/2270465703 >> 172.21.15.105:0/2270465703 conn(0x7f27980a2180 msgr2=0x7f279812f470 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:14.162 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.161+0000 7f279ebf0640 1 -- 172.21.15.105:0/2270465703 shutdown_connections 2024-09-21T01:46:14.163 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:14.161+0000 7f279ebf0640 1 -- 172.21.15.105:0/2270465703 wait complete. 2024-09-21T01:46:14.172 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705681 for osd.3 2024-09-21T01:46:14.172 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705681 for osd.2 2024-09-21T01:46:14.495 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:46:14.495+0000 7faf2995b640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-21T01:46:14.497 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:46:14.497+0000 7faf2995b640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-21T01:46:14.497 INFO:tasks.ceph.mgr.y.smithi105.stderr:2024-09-21T01:46:14.497+0000 7faf2995b640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-21T01:46:15.048 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T01:46:15.092 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T01:46:15.163 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.162+0000 7fbdc9c92640 1 -- 172.21.15.105:0/315454683 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbdc412f5d0 msgr2=0x7fbdc4130040 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:15.164 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.162+0000 7fbdc9c92640 1 --2- 172.21.15.105:0/315454683 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbdc412f5d0 0x7fbdc4130040 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7fbdac005fd0 tx=0x7fbdac02a410 comp rx=0 tx=0).stop 2024-09-21T01:46:15.164 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.162+0000 7fbdc9c92640 1 -- 172.21.15.105:0/315454683 shutdown_connections 2024-09-21T01:46:15.164 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.162+0000 7fbdc9c92640 1 --2- 172.21.15.105:0/315454683 >> v2:172.21.15.105:3300/0 conn(0x7fbdc412c0d0 0x7fbdc412c4e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.164 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.162+0000 7fbdc9c92640 1 --2- 172.21.15.105:0/315454683 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbdc412f5d0 0x7fbdc4130040 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.164 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.162+0000 7fbdc9c92640 1 --2- 172.21.15.105:0/315454683 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbdc40716c0 0x7fbdc412fb00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.164 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.162+0000 7fbdc9c92640 1 -- 172.21.15.105:0/315454683 >> 172.21.15.105:0/315454683 conn(0x7fbdc40a1e40 msgr2=0x7fbdc40a2250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:15.164 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.162+0000 7fbdc9c92640 1 -- 172.21.15.105:0/315454683 shutdown_connections 2024-09-21T01:46:15.164 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.163+0000 7fbdc9c92640 1 -- 172.21.15.105:0/315454683 wait complete. 2024-09-21T01:46:15.166 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.164+0000 7fbdc9c92640 1 Processor -- start 2024-09-21T01:46:15.166 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.164+0000 7fbdc9c92640 1 -- start start 2024-09-21T01:46:15.166 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.165+0000 7fbdc9c92640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbdc40716c0 0x7fbdc4134e60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:15.166 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.165+0000 7fbdc9c92640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbdc412c0d0 0x7fbdc41353a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:15.166 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.165+0000 7fbdc9c92640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbdc412f5d0 0x7fbdc41358e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:15.166 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.165+0000 7fbdc9c92640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7fbdc41373d0 con 0x7fbdc40716c0 2024-09-21T01:46:15.167 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.165+0000 7fbdc9c92640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7fbdc4137250 con 0x7fbdc412f5d0 2024-09-21T01:46:15.167 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.165+0000 7fbdc9c92640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7fbdc4137550 con 0x7fbdc412c0d0 2024-09-21T01:46:15.167 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.165+0000 7fbdc3fff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbdc412f5d0 0x7fbdc41358e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:15.167 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.165+0000 7fbdc37fe640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbdc40716c0 0x7fbdc4134e60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:15.167 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.165+0000 7fbdc3fff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbdc412f5d0 0x7fbdc41358e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:60002/0 (socket says 172.21.15.105:60002) 2024-09-21T01:46:15.167 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.165+0000 7fbdc3fff640 1 -- 172.21.15.105:0/1782612973 learned_addr learned my addr 172.21.15.105:0/1782612973 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:15.167 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.165+0000 7fbdc37fe640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbdc40716c0 0x7fbdc4134e60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:50604/0 (socket says 172.21.15.105:50604) 2024-09-21T01:46:15.167 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.165+0000 7fbdc2ffd640 1 --2- 172.21.15.105:0/1782612973 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbdc412c0d0 0x7fbdc41353a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:15.167 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.165+0000 7fbdc3fff640 1 -- 172.21.15.105:0/1782612973 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbdc412c0d0 msgr2=0x7fbdc41353a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:15.167 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.165+0000 7fbdc3fff640 1 --2- 172.21.15.105:0/1782612973 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbdc412c0d0 0x7fbdc41353a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.167 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.165+0000 7fbdc3fff640 1 -- 172.21.15.105:0/1782612973 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbdc40716c0 msgr2=0x7fbdc4134e60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:15.167 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.165+0000 7fbdc3fff640 1 --2- 172.21.15.105:0/1782612973 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbdc40716c0 0x7fbdc4134e60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.167 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.165+0000 7fbdc3fff640 1 -- 172.21.15.105:0/1782612973 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbdc4134ba0 con 0x7fbdc412f5d0 2024-09-21T01:46:15.168 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.165+0000 7fbdc3fff640 1 --2- 172.21.15.105:0/1782612973 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbdc412f5d0 0x7fbdc41358e0 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7fbdbc00efd0 tx=0x7fbdbc00c6a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:15.168 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.166+0000 7fbdbbfff640 1 -- 172.21.15.105:0/1782612973 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbdbc010630 con 0x7fbdc412f5d0 2024-09-21T01:46:15.168 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.166+0000 7fbdbbfff640 1 -- 172.21.15.105:0/1782612973 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fbdbc0108c0 con 0x7fbdc412f5d0 2024-09-21T01:46:15.168 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.166+0000 7fbdc9c92640 1 -- 172.21.15.105:0/1782612973 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbdc4134890 con 0x7fbdc412f5d0 2024-09-21T01:46:15.168 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.166+0000 7fbdbbfff640 1 -- 172.21.15.105:0/1782612973 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbdbc010cd0 con 0x7fbdc412f5d0 2024-09-21T01:46:15.168 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.166+0000 7fbdc9c92640 1 -- 172.21.15.105:0/1782612973 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fbdc409b4b0 con 0x7fbdc412f5d0 2024-09-21T01:46:15.168 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.167+0000 7fbdc9c92640 1 -- 172.21.15.105:0/1782612973 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbdc4130340 con 0x7fbdc412f5d0 2024-09-21T01:46:15.168 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.167+0000 7fbdbbfff640 1 -- 172.21.15.105:0/1782612973 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 8) ==== 63336+0+0 (secure 0 0 0) 0x7fbdbc01f020 con 0x7fbdc412f5d0 2024-09-21T01:46:15.169 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.167+0000 7fbdbbfff640 1 -- 172.21.15.105:0/1782612973 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7fbdbc0698e0 con 0x7fbdc412f5d0 2024-09-21T01:46:15.173 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T01:46:15.173 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T01:46:15.176 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.175+0000 7fbdbbfff640 1 -- 172.21.15.105:0/1782612973 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fbdc4130340 con 0x7fbdc412f5d0 2024-09-21T01:46:15.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.210+0000 7f024ce18640 1 -- 172.21.15.105:0/3758049465 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0248131f90 msgr2=0x7f02481343b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:15.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.210+0000 7f024ce18640 1 --2- 172.21.15.105:0/3758049465 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0248131f90 0x7f02481343b0 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7f0240009be0 tx=0x7f024002a800 comp rx=0 tx=0).stop 2024-09-21T01:46:15.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.210+0000 7f024ce18640 1 -- 172.21.15.105:0/3758049465 shutdown_connections 2024-09-21T01:46:15.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.210+0000 7f024ce18640 1 --2- 172.21.15.105:0/3758049465 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0248131f90 0x7f02481343b0 unknown :-1 s=CLOSED pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.210+0000 7f024ce18640 1 --2- 172.21.15.105:0/3758049465 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0248098200 0x7f0248131a50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.210+0000 7f024ce18640 1 -- 172.21.15.105:0/3758049465 >> 172.21.15.105:0/3758049465 conn(0x7f024809e6f0 msgr2=0x7f024809abc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:15.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.210+0000 7f024ce18640 1 -- 172.21.15.105:0/3758049465 shutdown_connections 2024-09-21T01:46:15.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.210+0000 7f024ce18640 1 -- 172.21.15.105:0/3758049465 wait complete. 2024-09-21T01:46:15.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.212+0000 7f024ce18640 1 Processor -- start 2024-09-21T01:46:15.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.212+0000 7f024ce18640 1 -- start start 2024-09-21T01:46:15.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.212+0000 7f024ce18640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f02480976e0 0x7f02480965d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:15.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.212+0000 7f024ce18640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0248098200 0x7f0248096b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:15.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.212+0000 7f024659c640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f02480976e0 0x7f02480965d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:15.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.212+0000 7f024659c640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f02480976e0 0x7f02480965d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:50172/0 (socket says 172.21.15.105:50172) 2024-09-21T01:46:15.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.213+0000 7f0245d9b640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0248098200 0x7f0248096b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:15.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.213+0000 7f0245d9b640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0248098200 0x7f0248096b10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:50612/0 (socket says 172.21.15.105:50612) 2024-09-21T01:46:15.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.213+0000 7f024ce18640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0248131f90 0x7f0248097050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:15.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.213+0000 7f024659c640 1 -- 172.21.15.105:0/580129878 learned_addr learned my addr 172.21.15.105:0/580129878 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:15.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.213+0000 7f024ce18640 1 -- 172.21.15.105:0/580129878 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f02481354f0 con 0x7f0248098200 2024-09-21T01:46:15.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.213+0000 7f024ce18640 1 -- 172.21.15.105:0/580129878 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f0248135370 con 0x7f0248131f90 2024-09-21T01:46:15.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.213+0000 7f024ce18640 1 -- 172.21.15.105:0/580129878 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f0248135670 con 0x7f02480976e0 2024-09-21T01:46:15.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.213+0000 7f0246d9d640 1 --2- 172.21.15.105:0/580129878 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0248131f90 0x7f0248097050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:15.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.213+0000 7f024659c640 1 -- 172.21.15.105:0/580129878 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0248131f90 msgr2=0x7f0248097050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:15.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.213+0000 7f024659c640 1 --2- 172.21.15.105:0/580129878 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0248131f90 0x7f0248097050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.213+0000 7f024659c640 1 -- 172.21.15.105:0/580129878 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0248098200 msgr2=0x7f0248096b10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:15.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.213+0000 7f024659c640 1 --2- 172.21.15.105:0/580129878 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0248098200 0x7f0248096b10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.213+0000 7f024659c640 1 -- 172.21.15.105:0/580129878 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0248073fa0 con 0x7f02480976e0 2024-09-21T01:46:15.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.213+0000 7f0245d9b640 1 --2- 172.21.15.105:0/580129878 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0248098200 0x7f0248096b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:15.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.213+0000 7f024659c640 1 --2- 172.21.15.105:0/580129878 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f02480976e0 0x7f02480965d0 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7f023800c400 tx=0x7f023800c800 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:15.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.213+0000 7f0236ffd640 1 -- 172.21.15.105:0/580129878 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0238017070 con 0x7f02480976e0 2024-09-21T01:46:15.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.214+0000 7f024ce18640 1 -- 172.21.15.105:0/580129878 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0248073110 con 0x7f02480976e0 2024-09-21T01:46:15.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.214+0000 7f0236ffd640 1 -- 172.21.15.105:0/580129878 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f02380053c0 con 0x7f02480976e0 2024-09-21T01:46:15.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.214+0000 7f0236ffd640 1 -- 172.21.15.105:0/580129878 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0238003850 con 0x7f02480976e0 2024-09-21T01:46:15.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.214+0000 7f024ce18640 1 -- 172.21.15.105:0/580129878 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f02481c5970 con 0x7f02480976e0 2024-09-21T01:46:15.217 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.215+0000 7f024ce18640 1 -- 172.21.15.105:0/580129878 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0248097af0 con 0x7f02480976e0 2024-09-21T01:46:15.217 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.215+0000 7f0236ffd640 1 -- 172.21.15.105:0/580129878 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 8) ==== 63336+0+0 (secure 0 0 0) 0x7f023801a070 con 0x7f02480976e0 2024-09-21T01:46:15.217 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.216+0000 7f0236ffd640 1 -- 172.21.15.105:0/580129878 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7f0238068010 con 0x7f02480976e0 2024-09-21T01:46:15.223 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.223+0000 7f0236ffd640 1 -- 172.21.15.105:0/580129878 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f0238066ac0 con 0x7f02480976e0 2024-09-21T01:46:15.291 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.290+0000 7ff0c351f640 1 -- 172.21.15.105:0/2441831095 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff0bc12f770 msgr2=0x7ff0bc071730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:15.291 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.291+0000 7ff0c351f640 1 --2- 172.21.15.105:0/2441831095 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff0bc12f770 0x7ff0bc071730 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7ff0b8009830 tx=0x7ff0b802a420 comp rx=0 tx=0).stop 2024-09-21T01:46:15.291 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.291+0000 7ff0c351f640 1 -- 172.21.15.105:0/2441831095 shutdown_connections 2024-09-21T01:46:15.292 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.291+0000 7ff0c351f640 1 --2- 172.21.15.105:0/2441831095 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff0bc070c50 0x7ff0bc12c060 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.292 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.291+0000 7ff0c351f640 1 --2- 172.21.15.105:0/2441831095 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff0bc12f770 0x7ff0bc071730 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.292 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.291+0000 7ff0c351f640 1 -- 172.21.15.105:0/2441831095 >> 172.21.15.105:0/2441831095 conn(0x7ff0bc0a2080 msgr2=0x7ff0bc0a2490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:15.292 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.291+0000 7ff0c351f640 1 -- 172.21.15.105:0/2441831095 shutdown_connections 2024-09-21T01:46:15.292 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.291+0000 7ff0c351f640 1 -- 172.21.15.105:0/2441831095 wait complete. 2024-09-21T01:46:15.293 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.291+0000 7fad84818640 1 -- 172.21.15.105:0/1488300540 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fad80123a90 msgr2=0x7fad80123ea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:15.293 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.291+0000 7fad84818640 1 --2- 172.21.15.105:0/1488300540 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fad80123a90 0x7fad80123ea0 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7fad70009830 tx=0x7fad7002a480 comp rx=0 tx=0).stop 2024-09-21T01:46:15.293 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.292+0000 7fad84818640 1 -- 172.21.15.105:0/1488300540 shutdown_connections 2024-09-21T01:46:15.293 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.292+0000 7fad84818640 1 --2- 172.21.15.105:0/1488300540 >> v2:172.21.15.105:3300/0 conn(0x7fad800985f0 0x7fad80098a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.293 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.292+0000 7fad84818640 1 --2- 172.21.15.105:0/1488300540 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fad801243e0 0x7fad800980b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.293 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.292+0000 7fad84818640 1 --2- 172.21.15.105:0/1488300540 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fad80123a90 0x7fad80123ea0 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.293 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.292+0000 7fad84818640 1 -- 172.21.15.105:0/1488300540 >> 172.21.15.105:0/1488300540 conn(0x7fad8009f000 msgr2=0x7fad8009b4d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:15.293 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.292+0000 7fad84818640 1 -- 172.21.15.105:0/1488300540 shutdown_connections 2024-09-21T01:46:15.293 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.292+0000 7fad84818640 1 -- 172.21.15.105:0/1488300540 wait complete. 2024-09-21T01:46:15.294 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.292+0000 7ff0c351f640 1 Processor -- start 2024-09-21T01:46:15.294 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.292+0000 7ff0c351f640 1 -- start start 2024-09-21T01:46:15.294 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.293+0000 7ff0c351f640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff0bc070c50 0x7ff0bc1c5b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:15.294 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.293+0000 7ff0c351f640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff0bc071060 0x7ff0bc1c60b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:15.294 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.293+0000 7ff0c351f640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff0bc12f770 0x7ff0bc1c65f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:15.294 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.293+0000 7ff0c351f640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7ff0bc138700 con 0x7ff0bc12f770 2024-09-21T01:46:15.295 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.293+0000 7ff0c351f640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7ff0bc138580 con 0x7ff0bc071060 2024-09-21T01:46:15.295 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.293+0000 7ff0c351f640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7ff0bc138880 con 0x7ff0bc070c50 2024-09-21T01:46:15.295 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.293+0000 7ff0c12bb640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff0bc070c50 0x7ff0bc1c5b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:15.295 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.293+0000 7ff0c12bb640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff0bc070c50 0x7ff0bc1c5b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:50202/0 (socket says 172.21.15.105:50202) 2024-09-21T01:46:15.295 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.293+0000 7ff0c0aba640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff0bc071060 0x7ff0bc1c60b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:15.295 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.293+0000 7ff0c12bb640 1 -- 172.21.15.105:0/3805588301 learned_addr learned my addr 172.21.15.105:0/3805588301 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:15.295 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.293+0000 7fad84818640 1 Processor -- start 2024-09-21T01:46:15.295 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.293+0000 7fad84818640 1 -- start start 2024-09-21T01:46:15.296 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.293+0000 7ff0c1abc640 1 --2- 172.21.15.105:0/3805588301 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff0bc12f770 0x7ff0bc1c65f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:15.296 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.293+0000 7ff0c1abc640 1 -- 172.21.15.105:0/3805588301 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff0bc070c50 msgr2=0x7ff0bc1c5b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:15.296 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.293+0000 7ff0c1abc640 1 --2- 172.21.15.105:0/3805588301 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff0bc070c50 0x7ff0bc1c5b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.296 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.293+0000 7ff0c1abc640 1 -- 172.21.15.105:0/3805588301 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff0bc071060 msgr2=0x7ff0bc1c60b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:15.296 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.293+0000 7ff0c1abc640 1 --2- 172.21.15.105:0/3805588301 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff0bc071060 0x7ff0bc1c60b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.296 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.293+0000 7ff0c1abc640 1 -- 172.21.15.105:0/3805588301 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff0bc1328c0 con 0x7ff0bc12f770 2024-09-21T01:46:15.296 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.293+0000 7ff0c0aba640 1 --2- 172.21.15.105:0/3805588301 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff0bc071060 0x7ff0bc1c60b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:15.296 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.294+0000 7ff0c1abc640 1 --2- 172.21.15.105:0/3805588301 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff0bc12f770 0x7ff0bc1c65f0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7ff0b400cbc0 tx=0x7ff0b400a430 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:15.297 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.294+0000 7fad84818640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fad800985f0 0x7fad80071860 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:15.297 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.294+0000 7fad84818640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fad80123a90 0x7fad80071da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:15.297 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.294+0000 7fad84818640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fad801243e0 0x7fad800722e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:15.297 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.294+0000 7fad84818640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7fad80135370 con 0x7fad800985f0 2024-09-21T01:46:15.297 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.294+0000 7fad84818640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7fad801351f0 con 0x7fad801243e0 2024-09-21T01:46:15.297 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.294+0000 7fad84818640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7fad801354f0 con 0x7fad80123a90 2024-09-21T01:46:15.297 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.294+0000 7ff0b1ffb640 1 -- 172.21.15.105:0/3805588301 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff0b4016020 con 0x7ff0bc12f770 2024-09-21T01:46:15.299 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.294+0000 7fad7e59c640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fad800985f0 0x7fad80071860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:15.299 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.294+0000 7fad7e59c640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fad800985f0 0x7fad80071860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:50642/0 (socket says 172.21.15.105:50642) 2024-09-21T01:46:15.299 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.294+0000 7fad7e59c640 1 -- 172.21.15.105:0/3159522745 learned_addr learned my addr 172.21.15.105:0/3159522745 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:15.299 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.294+0000 7fad7dd9b640 1 --2- 172.21.15.105:0/3159522745 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fad80123a90 0x7fad80071da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:15.299 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.294+0000 7fad7ed9d640 1 --2- 172.21.15.105:0/3159522745 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fad801243e0 0x7fad800722e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:15.299 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.294+0000 7fad7e59c640 1 -- 172.21.15.105:0/3159522745 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fad80123a90 msgr2=0x7fad80071da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:15.299 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.294+0000 7fad7e59c640 1 --2- 172.21.15.105:0/3159522745 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fad80123a90 0x7fad80071da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.299 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.294+0000 7fad7e59c640 1 -- 172.21.15.105:0/3159522745 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fad801243e0 msgr2=0x7fad800722e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:15.299 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.295+0000 7fad7e59c640 1 --2- 172.21.15.105:0/3159522745 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fad801243e0 0x7fad800722e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.299 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.295+0000 7fad7e59c640 1 -- 172.21.15.105:0/3159522745 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fad80134620 con 0x7fad800985f0 2024-09-21T01:46:15.299 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.295+0000 7fad7ed9d640 1 --2- 172.21.15.105:0/3159522745 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fad801243e0 0x7fad800722e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T01:46:15.299 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.295+0000 7fad7e59c640 1 --2- 172.21.15.105:0/3159522745 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fad800985f0 0x7fad80071860 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fad70009950 tx=0x7fad700316c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:15.299 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.295+0000 7fad6effd640 1 -- 172.21.15.105:0/3159522745 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fad70040070 con 0x7fad800985f0 2024-09-21T01:46:15.299 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.295+0000 7fad6effd640 1 -- 172.21.15.105:0/3159522745 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fad700038e0 con 0x7fad800985f0 2024-09-21T01:46:15.300 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.295+0000 7fad84818640 1 -- 172.21.15.105:0/3159522745 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fad801c57f0 con 0x7fad800985f0 2024-09-21T01:46:15.300 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.295+0000 7fad6effd640 1 -- 172.21.15.105:0/3159522745 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fad70005c90 con 0x7fad800985f0 2024-09-21T01:46:15.300 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.294+0000 7ff0c351f640 1 -- 172.21.15.105:0/3805588301 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff0bc134000 con 0x7ff0bc12f770 2024-09-21T01:46:15.300 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.294+0000 7ff0b1ffb640 1 -- 172.21.15.105:0/3805588301 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7ff0b400ae30 con 0x7ff0bc12f770 2024-09-21T01:46:15.300 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.294+0000 7ff0b1ffb640 1 -- 172.21.15.105:0/3805588301 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff0b4012680 con 0x7ff0bc12f770 2024-09-21T01:46:15.300 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.294+0000 7ff0c351f640 1 -- 172.21.15.105:0/3805588301 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff0bc131700 con 0x7ff0bc12f770 2024-09-21T01:46:15.300 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.296+0000 7ff0a77fe640 1 -- 172.21.15.105:0/3805588301 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff094005290 con 0x7ff0bc12f770 2024-09-21T01:46:15.301 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.295+0000 7fad84818640 1 -- 172.21.15.105:0/3159522745 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fad801c5c90 con 0x7fad800985f0 2024-09-21T01:46:15.301 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.297+0000 7ff0b1ffb640 1 -- 172.21.15.105:0/3805588301 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 8) ==== 63336+0+0 (secure 0 0 0) 0x7ff0b4019020 con 0x7ff0bc12f770 2024-09-21T01:46:15.301 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.297+0000 7fad84818640 1 -- 172.21.15.105:0/3159522745 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fad801c6620 con 0x7fad800985f0 2024-09-21T01:46:15.301 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.297+0000 7fad6effd640 1 -- 172.21.15.105:0/3159522745 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 8) ==== 63336+0+0 (secure 0 0 0) 0x7fad70039040 con 0x7fad800985f0 2024-09-21T01:46:15.301 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.297+0000 7fad6effd640 1 -- 172.21.15.105:0/3159522745 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7fad7008a1d0 con 0x7fad800985f0 2024-09-21T01:46:15.302 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.297+0000 7ff0b1ffb640 1 -- 172.21.15.105:0/3805588301 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7ff0b40673d0 con 0x7ff0bc12f770 2024-09-21T01:46:15.305 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.305+0000 7ff0b1ffb640 1 -- 172.21.15.105:0/3805588301 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7ff0b40413a0 con 0x7ff0bc12f770 2024-09-21T01:46:15.313 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.313+0000 7fad6effd640 1 -- 172.21.15.105:0/3159522745 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fad801c6620 con 0x7fad800985f0 2024-09-21T01:46:15.376 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.375+0000 7fbdc9c92640 1 -- 172.21.15.105:0/1782612973 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7fbdc412c4e0 con 0x7fbdc412f5d0 2024-09-21T01:46:15.376 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.376+0000 7fbdbbfff640 1 -- 172.21.15.105:0/1782612973 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fbdc4134ba0 con 0x7fbdc412f5d0 2024-09-21T01:46:15.376 INFO:teuthology.orchestra.run.smithi105.stdout:38654705682 2024-09-21T01:46:15.378 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.378+0000 7fbdc9c92640 1 -- 172.21.15.105:0/1782612973 >> v1:172.21.15.105:6816/2838005468 conn(0x7fbda4051300 legacy=0x7fbda4053810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:15.379 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.378+0000 7fbdc9c92640 1 -- 172.21.15.105:0/1782612973 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbdc412f5d0 msgr2=0x7fbdc41358e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:15.379 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.378+0000 7fbdc9c92640 1 --2- 172.21.15.105:0/1782612973 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbdc412f5d0 0x7fbdc41358e0 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7fbdbc00efd0 tx=0x7fbdbc00c6a0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.379 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.378+0000 7fbdc9c92640 1 -- 172.21.15.105:0/1782612973 shutdown_connections 2024-09-21T01:46:15.379 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.378+0000 7fbdc9c92640 1 --2- 172.21.15.105:0/1782612973 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbdc412f5d0 0x7fbdc41358e0 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.379 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.378+0000 7fbdc9c92640 1 --2- 172.21.15.105:0/1782612973 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbdc412c0d0 0x7fbdc41353a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.379 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.378+0000 7fbdc9c92640 1 --2- 172.21.15.105:0/1782612973 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbdc40716c0 0x7fbdc4134e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.379 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.378+0000 7fbdc9c92640 1 -- 172.21.15.105:0/1782612973 >> 172.21.15.105:0/1782612973 conn(0x7fbdc40a1e40 msgr2=0x7fbdc40731e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:15.379 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.378+0000 7fbdc9c92640 1 -- 172.21.15.105:0/1782612973 shutdown_connections 2024-09-21T01:46:15.379 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.378+0000 7fbdc9c92640 1 -- 172.21.15.105:0/1782612973 wait complete. 2024-09-21T01:46:15.390 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705684 got 38654705682 for osd.0 2024-09-21T01:46:15.425 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.425+0000 7f024ce18640 1 -- 172.21.15.105:0/580129878 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7f02480743f0 con 0x7f02480976e0 2024-09-21T01:46:15.425 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.425+0000 7f0236ffd640 1 -- 172.21.15.105:0/580129878 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f023806a130 con 0x7f02480976e0 2024-09-21T01:46:15.426 INFO:teuthology.orchestra.run.smithi105.stdout:38654705681 2024-09-21T01:46:15.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.427+0000 7f024ce18640 1 -- 172.21.15.105:0/580129878 >> v1:172.21.15.105:6816/2838005468 conn(0x7f0228051330 legacy=0x7f0228053840 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:15.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.427+0000 7f024ce18640 1 -- 172.21.15.105:0/580129878 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f02480976e0 msgr2=0x7f02480965d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:15.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.427+0000 7f024ce18640 1 --2- 172.21.15.105:0/580129878 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f02480976e0 0x7f02480965d0 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7f023800c400 tx=0x7f023800c800 comp rx=0 tx=0).stop 2024-09-21T01:46:15.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.427+0000 7f024ce18640 1 -- 172.21.15.105:0/580129878 shutdown_connections 2024-09-21T01:46:15.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.427+0000 7f024ce18640 1 --2- 172.21.15.105:0/580129878 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0248131f90 0x7f0248097050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.427+0000 7f024ce18640 1 --2- 172.21.15.105:0/580129878 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0248098200 0x7f0248096b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.427+0000 7f024ce18640 1 --2- 172.21.15.105:0/580129878 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f02480976e0 0x7f02480965d0 unknown :-1 s=CLOSED pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.427+0000 7f024ce18640 1 -- 172.21.15.105:0/580129878 >> 172.21.15.105:0/580129878 conn(0x7f024809e6f0 msgr2=0x7f024809aad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:15.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.427+0000 7f024ce18640 1 -- 172.21.15.105:0/580129878 shutdown_connections 2024-09-21T01:46:15.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.427+0000 7f024ce18640 1 -- 172.21.15.105:0/580129878 wait complete. 2024-09-21T01:46:15.438 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705681 for osd.1 2024-09-21T01:46:15.482 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.482+0000 7ff0a77fe640 1 -- 172.21.15.105:0/3805588301 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7ff094005580 con 0x7ff0bc12f770 2024-09-21T01:46:15.483 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.482+0000 7ff0b1ffb640 1 -- 172.21.15.105:0/3805588301 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7ff0b4069b90 con 0x7ff0bc12f770 2024-09-21T01:46:15.483 INFO:teuthology.orchestra.run.smithi105.stdout:38654705681 2024-09-21T01:46:15.485 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.484+0000 7ff0a77fe640 1 -- 172.21.15.105:0/3805588301 >> v1:172.21.15.105:6816/2838005468 conn(0x7ff0a0051300 legacy=0x7ff0a0053810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:15.485 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.484+0000 7ff0a77fe640 1 -- 172.21.15.105:0/3805588301 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff0bc12f770 msgr2=0x7ff0bc1c65f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:15.485 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.484+0000 7ff0a77fe640 1 --2- 172.21.15.105:0/3805588301 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff0bc12f770 0x7ff0bc1c65f0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7ff0b400cbc0 tx=0x7ff0b400a430 comp rx=0 tx=0).stop 2024-09-21T01:46:15.485 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.484+0000 7ff0a77fe640 1 -- 172.21.15.105:0/3805588301 shutdown_connections 2024-09-21T01:46:15.485 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.484+0000 7ff0a77fe640 1 --2- 172.21.15.105:0/3805588301 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff0bc12f770 0x7ff0bc1c65f0 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.485 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.484+0000 7ff0a77fe640 1 --2- 172.21.15.105:0/3805588301 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff0bc071060 0x7ff0bc1c60b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.485 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.484+0000 7ff0a77fe640 1 --2- 172.21.15.105:0/3805588301 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff0bc070c50 0x7ff0bc1c5b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.485 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.484+0000 7ff0a77fe640 1 -- 172.21.15.105:0/3805588301 >> 172.21.15.105:0/3805588301 conn(0x7ff0bc0a2080 msgr2=0x7ff0bc12c190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:15.485 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.484+0000 7ff0a77fe640 1 -- 172.21.15.105:0/3805588301 shutdown_connections 2024-09-21T01:46:15.485 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.484+0000 7ff0a77fe640 1 -- 172.21.15.105:0/3805588301 wait complete. 2024-09-21T01:46:15.490 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.490+0000 7fad84818640 1 -- 172.21.15.105:0/3159522745 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7fad80123ea0 con 0x7fad800985f0 2024-09-21T01:46:15.493 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.493+0000 7fad6effd640 1 -- 172.21.15.105:0/3159522745 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fad7003fda0 con 0x7fad800985f0 2024-09-21T01:46:15.493 INFO:teuthology.orchestra.run.smithi105.stdout:38654705684 2024-09-21T01:46:15.496 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.495+0000 7fad84818640 1 -- 172.21.15.105:0/3159522745 >> v1:172.21.15.105:6816/2838005468 conn(0x7fad60051300 legacy=0x7fad60053810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:15.496 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.495+0000 7fad84818640 1 -- 172.21.15.105:0/3159522745 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fad800985f0 msgr2=0x7fad80071860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:15.496 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.495+0000 7fad84818640 1 --2- 172.21.15.105:0/3159522745 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fad800985f0 0x7fad80071860 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fad70009950 tx=0x7fad700316c0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.496 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.495+0000 7fad84818640 1 -- 172.21.15.105:0/3159522745 shutdown_connections 2024-09-21T01:46:15.496 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.495+0000 7fad84818640 1 --2- 172.21.15.105:0/3159522745 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fad801243e0 0x7fad800722e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.496 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.495+0000 7fad84818640 1 --2- 172.21.15.105:0/3159522745 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fad80123a90 0x7fad80071da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.496 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.495+0000 7fad84818640 1 --2- 172.21.15.105:0/3159522745 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fad800985f0 0x7fad80071860 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:15.496 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.495+0000 7fad84818640 1 -- 172.21.15.105:0/3159522745 >> 172.21.15.105:0/3159522745 conn(0x7fad8009f000 msgr2=0x7fad8012b3c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:15.496 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.495+0000 7fad84818640 1 -- 172.21.15.105:0/3159522745 shutdown_connections 2024-09-21T01:46:15.496 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:15.495+0000 7fad84818640 1 -- 172.21.15.105:0/3159522745 wait complete. 2024-09-21T01:46:15.497 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705681 for osd.2 2024-09-21T01:46:15.507 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705684 for osd.3 2024-09-21T01:46:15.507 DEBUG:teuthology.parallel:result is None 2024-09-21T01:46:16.390 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T01:46:16.439 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T01:46:16.498 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T01:46:16.508 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.508+0000 7f8f023eb640 1 -- 172.21.15.105:0/130266318 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8efc073960 msgr2=0x7f8efc129ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:16.508 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.508+0000 7f8f023eb640 1 --2- 172.21.15.105:0/130266318 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8efc073960 0x7f8efc129ec0 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 crypto rx=0x7f8eec009830 tx=0x7f8eec02a3f0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.508 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.508+0000 7f8f023eb640 1 -- 172.21.15.105:0/130266318 shutdown_connections 2024-09-21T01:46:16.509 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.508+0000 7f8f023eb640 1 --2- 172.21.15.105:0/130266318 >> v2:172.21.15.105:3300/0 conn(0x7f8efc12d4a0 0x7f8efc12d8b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.509 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.508+0000 7f8f023eb640 1 --2- 172.21.15.105:0/130266318 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8efc12a400 0x7f8efc12a810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.509 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.508+0000 7f8f023eb640 1 --2- 172.21.15.105:0/130266318 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8efc073960 0x7f8efc129ec0 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.509 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.508+0000 7f8f023eb640 1 -- 172.21.15.105:0/130266318 >> 172.21.15.105:0/130266318 conn(0x7f8efc09fc30 msgr2=0x7f8efc0a0040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:16.509 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.508+0000 7f8f023eb640 1 -- 172.21.15.105:0/130266318 shutdown_connections 2024-09-21T01:46:16.509 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.508+0000 7f8f023eb640 1 -- 172.21.15.105:0/130266318 wait complete. 2024-09-21T01:46:16.509 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.509+0000 7f8f023eb640 1 Processor -- start 2024-09-21T01:46:16.510 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.509+0000 7f8f023eb640 1 -- start start 2024-09-21T01:46:16.511 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.510+0000 7f8f023eb640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8efc073960 0x7f8efc131ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:16.511 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.510+0000 7f8f023eb640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8efc12a400 0x7f8efc132220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:16.511 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.510+0000 7f8f023eb640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8efc12d4a0 0x7f8efc1c3880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:16.511 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.510+0000 7f8f023eb640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f8efc1351a0 con 0x7f8efc12d4a0 2024-09-21T01:46:16.511 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.510+0000 7f8f023eb640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f8efc135020 con 0x7f8efc12a400 2024-09-21T01:46:16.511 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.510+0000 7f8f023eb640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f8efc135320 con 0x7f8efc073960 2024-09-21T01:46:16.511 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.510+0000 7f8efb7fe640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8efc12a400 0x7f8efc132220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:16.511 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.510+0000 7f8f00988640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8efc12d4a0 0x7f8efc1c3880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:16.511 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.510+0000 7f8efb7fe640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8efc12a400 0x7f8efc132220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:60078/0 (socket says 172.21.15.105:60078) 2024-09-21T01:46:16.511 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.510+0000 7f8f00988640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8efc12d4a0 0x7f8efc1c3880 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:50678/0 (socket says 172.21.15.105:50678) 2024-09-21T01:46:16.511 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.510+0000 7f8efb7fe640 1 -- 172.21.15.105:0/774557556 learned_addr learned my addr 172.21.15.105:0/774557556 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:16.512 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.510+0000 7f8f00988640 1 -- 172.21.15.105:0/774557556 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8efc073960 msgr2=0x7f8efc131ce0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:16.513 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.511+0000 7f8efbfff640 1 --2- 172.21.15.105:0/774557556 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8efc073960 0x7f8efc131ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:16.513 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.511+0000 7f8f00988640 1 --2- 172.21.15.105:0/774557556 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8efc073960 0x7f8efc131ce0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.513 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.511+0000 7f8f00988640 1 -- 172.21.15.105:0/774557556 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8efc12a400 msgr2=0x7f8efc132220 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:16.513 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.511+0000 7f8f00988640 1 --2- 172.21.15.105:0/774557556 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8efc12a400 0x7f8efc132220 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.513 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.511+0000 7f8f00988640 1 -- 172.21.15.105:0/774557556 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8efc133e20 con 0x7f8efc12d4a0 2024-09-21T01:46:16.514 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.511+0000 7f8efb7fe640 1 --2- 172.21.15.105:0/774557556 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8efc12a400 0x7f8efc132220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:16.514 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.511+0000 7f8efbfff640 1 --2- 172.21.15.105:0/774557556 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8efc073960 0x7f8efc131ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T01:46:16.514 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.511+0000 7f8f00988640 1 --2- 172.21.15.105:0/774557556 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8efc12d4a0 0x7f8efc1c3880 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f8ef400eb50 tx=0x7f8ef400c6a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:16.514 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.511+0000 7f8ef8ff9640 1 -- 172.21.15.105:0/774557556 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8ef4003b70 con 0x7f8efc12d4a0 2024-09-21T01:46:16.514 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.511+0000 7f8ef8ff9640 1 -- 172.21.15.105:0/774557556 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f8ef4003d10 con 0x7f8efc12d4a0 2024-09-21T01:46:16.514 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.511+0000 7f8ef8ff9640 1 -- 172.21.15.105:0/774557556 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8ef4004280 con 0x7f8efc12d4a0 2024-09-21T01:46:16.514 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.511+0000 7f8f023eb640 1 -- 172.21.15.105:0/774557556 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8efc132760 con 0x7f8efc12d4a0 2024-09-21T01:46:16.514 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.512+0000 7f8f023eb640 1 -- 172.21.15.105:0/774557556 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f8efc1c3fa0 con 0x7f8efc12d4a0 2024-09-21T01:46:16.514 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.513+0000 7f8ef8ff9640 1 -- 172.21.15.105:0/774557556 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 9) ==== 63442+0+0 (secure 0 0 0) 0x7f8ef4004420 con 0x7f8efc12d4a0 2024-09-21T01:46:16.514 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.513+0000 7f8f023eb640 1 -- 172.21.15.105:0/774557556 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8efc1334e0 con 0x7f8efc12d4a0 2024-09-21T01:46:16.514 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.513+0000 7f8ef8ff9640 1 -- 172.21.15.105:0/774557556 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7f8ef4069a70 con 0x7f8efc12d4a0 2024-09-21T01:46:16.518 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.518+0000 7f8ef8ff9640 1 -- 172.21.15.105:0/774557556 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f8ef400ee50 con 0x7f8efc12d4a0 2024-09-21T01:46:16.557 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.556+0000 7fd4697eb640 1 -- 172.21.15.105:0/1920229330 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd464073dd0 msgr2=0x7fd46412b4e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:16.557 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.556+0000 7fd4697eb640 1 --2- 172.21.15.105:0/1920229330 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd464073dd0 0x7fd46412b4e0 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7fd454009830 tx=0x7fd45402a480 comp rx=0 tx=0).stop 2024-09-21T01:46:16.557 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.556+0000 7fd4697eb640 1 -- 172.21.15.105:0/1920229330 shutdown_connections 2024-09-21T01:46:16.557 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.556+0000 7fd4697eb640 1 --2- 172.21.15.105:0/1920229330 >> v2:172.21.15.105:3300/0 conn(0x7fd46412ba20 0x7fd46412be30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.557 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.556+0000 7fd4697eb640 1 --2- 172.21.15.105:0/1920229330 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd464073dd0 0x7fd46412b4e0 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.557 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.556+0000 7fd4697eb640 1 -- 172.21.15.105:0/1920229330 >> 172.21.15.105:0/1920229330 conn(0x7fd4640a0160 msgr2=0x7fd46409c630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:16.557 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.556+0000 7fd4697eb640 1 -- 172.21.15.105:0/1920229330 shutdown_connections 2024-09-21T01:46:16.557 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.556+0000 7fd4697eb640 1 -- 172.21.15.105:0/1920229330 wait complete. 2024-09-21T01:46:16.558 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.558+0000 7fd4697eb640 1 Processor -- start 2024-09-21T01:46:16.558 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.558+0000 7fd4697eb640 1 -- start start 2024-09-21T01:46:16.560 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.558+0000 7fd4697eb640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd464073dd0 0x7fd46406faf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:16.560 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.558+0000 7fd4697eb640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd4640990c0 0x7fd464070030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:16.560 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.558+0000 7fd4697eb640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd46412ba20 0x7fd464070570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:16.560 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.558+0000 7fd4697eb640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7fd464136680 con 0x7fd46412ba20 2024-09-21T01:46:16.560 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.558+0000 7fd4697eb640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7fd464136500 con 0x7fd4640990c0 2024-09-21T01:46:16.560 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.558+0000 7fd4697eb640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7fd464136800 con 0x7fd464073dd0 2024-09-21T01:46:16.560 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.558+0000 7fd4627fc640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd4640990c0 0x7fd464070030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:16.560 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.558+0000 7fd4637fe640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd46412ba20 0x7fd464070570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:16.560 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.558+0000 7fd462ffd640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd464073dd0 0x7fd46406faf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:16.560 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.558+0000 7fd4627fc640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd4640990c0 0x7fd464070030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:60084/0 (socket says 172.21.15.105:60084) 2024-09-21T01:46:16.560 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.558+0000 7fd4627fc640 1 -- 172.21.15.105:0/3462924711 learned_addr learned my addr 172.21.15.105:0/3462924711 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:16.560 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.559+0000 7fd4627fc640 1 -- 172.21.15.105:0/3462924711 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd464073dd0 msgr2=0x7fd46406faf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:16.560 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.559+0000 7fd4627fc640 1 --2- 172.21.15.105:0/3462924711 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd464073dd0 0x7fd46406faf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.561 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.559+0000 7fd4627fc640 1 -- 172.21.15.105:0/3462924711 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd46412ba20 msgr2=0x7fd464070570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:16.561 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.559+0000 7fd4627fc640 1 --2- 172.21.15.105:0/3462924711 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd46412ba20 0x7fd464070570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.561 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.559+0000 7fd4627fc640 1 -- 172.21.15.105:0/3462924711 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd464071a50 con 0x7fd4640990c0 2024-09-21T01:46:16.561 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.559+0000 7fd4637fe640 1 --2- 172.21.15.105:0/3462924711 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd46412ba20 0x7fd464070570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:16.561 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.559+0000 7fd4627fc640 1 --2- 172.21.15.105:0/3462924711 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd4640990c0 0x7fd464070030 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7fd45800e9f0 tx=0x7fd45800eeb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:16.561 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.559+0000 7fd44b7fe640 1 -- 172.21.15.105:0/3462924711 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd45800cd80 con 0x7fd4640990c0 2024-09-21T01:46:16.561 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.559+0000 7fd44b7fe640 1 -- 172.21.15.105:0/3462924711 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fd458004020 con 0x7fd4640990c0 2024-09-21T01:46:16.561 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.559+0000 7fd4697eb640 1 -- 172.21.15.105:0/3462924711 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd464071530 con 0x7fd4640990c0 2024-09-21T01:46:16.561 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.559+0000 7fd44b7fe640 1 -- 172.21.15.105:0/3462924711 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd458004470 con 0x7fd4640990c0 2024-09-21T01:46:16.561 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.559+0000 7fd4697eb640 1 -- 172.21.15.105:0/3462924711 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fd46412f940 con 0x7fd4640990c0 2024-09-21T01:46:16.562 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.560+0000 7fd44b7fe640 1 -- 172.21.15.105:0/3462924711 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 9) ==== 63442+0+0 (secure 0 0 0) 0x7fd4580035f0 con 0x7fd4640990c0 2024-09-21T01:46:16.562 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.561+0000 7fd4697eb640 1 -- 172.21.15.105:0/3462924711 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd46412b4e0 con 0x7fd4640990c0 2024-09-21T01:46:16.562 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.561+0000 7fd44b7fe640 1 -- 172.21.15.105:0/3462924711 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7fd458069900 con 0x7fd4640990c0 2024-09-21T01:46:16.565 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.565+0000 7fd44b7fe640 1 -- 172.21.15.105:0/3462924711 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fd458009490 con 0x7fd4640990c0 2024-09-21T01:46:16.615 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.614+0000 7f39f7bf5640 1 -- 172.21.15.105:0/3291037342 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f39f0134e80 msgr2=0x7f39f01372a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:16.615 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.614+0000 7f39f7bf5640 1 --2- 172.21.15.105:0/3291037342 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f39f0134e80 0x7f39f01372a0 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7f39e8005fd0 tx=0x7f39e802aa10 comp rx=0 tx=0).stop 2024-09-21T01:46:16.615 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.614+0000 7f39f7bf5640 1 -- 172.21.15.105:0/3291037342 shutdown_connections 2024-09-21T01:46:16.615 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.614+0000 7f39f7bf5640 1 --2- 172.21.15.105:0/3291037342 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f39f0134e80 0x7f39f01372a0 unknown :-1 s=CLOSED pgs=34 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.615 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.614+0000 7f39f7bf5640 1 --2- 172.21.15.105:0/3291037342 >> v2:172.21.15.105:3300/0 conn(0x7f39f0071730 0x7f39f0134940 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.615 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.614+0000 7f39f7bf5640 1 -- 172.21.15.105:0/3291037342 >> 172.21.15.105:0/3291037342 conn(0x7f39f00a2080 msgr2=0x7f39f00a2490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:16.615 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.614+0000 7f39f7bf5640 1 -- 172.21.15.105:0/3291037342 shutdown_connections 2024-09-21T01:46:16.615 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.614+0000 7f39f7bf5640 1 -- 172.21.15.105:0/3291037342 wait complete. 2024-09-21T01:46:16.616 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.615+0000 7f39f7bf5640 1 Processor -- start 2024-09-21T01:46:16.616 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.615+0000 7f39f7bf5640 1 -- start start 2024-09-21T01:46:16.617 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.616+0000 7f39f7bf5640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f39f0071730 0x7f39f01c5af0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:16.618 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.616+0000 7f39f7bf5640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f39f012f770 0x7f39f01c6030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:16.618 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.616+0000 7f39f7bf5640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f39f0134e80 0x7f39f01c6570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:16.618 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.616+0000 7f39f7bf5640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f39f01386a0 con 0x7f39f012f770 2024-09-21T01:46:16.618 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.616+0000 7f39f7bf5640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f39f0138520 con 0x7f39f0071730 2024-09-21T01:46:16.618 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.616+0000 7f39f7bf5640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f39f0138820 con 0x7f39f0134e80 2024-09-21T01:46:16.618 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.616+0000 7f39f5190640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f39f012f770 0x7f39f01c6030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:16.618 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.616+0000 7f39f5991640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f39f0071730 0x7f39f01c5af0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:16.618 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.616+0000 7f39f5190640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f39f012f770 0x7f39f01c6030 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:50704/0 (socket says 172.21.15.105:50704) 2024-09-21T01:46:16.618 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.616+0000 7f39f5190640 1 -- 172.21.15.105:0/2341899414 learned_addr learned my addr 172.21.15.105:0/2341899414 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:16.618 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.616+0000 7f39f6192640 1 --2- 172.21.15.105:0/2341899414 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f39f0134e80 0x7f39f01c6570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:16.618 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.616+0000 7f39f5190640 1 -- 172.21.15.105:0/2341899414 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f39f0134e80 msgr2=0x7f39f01c6570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:16.618 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.616+0000 7f39f5190640 1 --2- 172.21.15.105:0/2341899414 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f39f0134e80 0x7f39f01c6570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.618 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.616+0000 7f39f5190640 1 -- 172.21.15.105:0/2341899414 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f39f0071730 msgr2=0x7f39f01c5af0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:16.619 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.616+0000 7f39f5190640 1 --2- 172.21.15.105:0/2341899414 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f39f0071730 0x7f39f01c5af0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.619 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.616+0000 7f39f5190640 1 -- 172.21.15.105:0/2341899414 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f39f0130e70 con 0x7f39f012f770 2024-09-21T01:46:16.619 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.617+0000 7f39f5190640 1 --2- 172.21.15.105:0/2341899414 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f39f012f770 0x7f39f01c6030 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f39e000ea20 tx=0x7f39e000eee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:16.619 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.617+0000 7f39e67fc640 1 -- 172.21.15.105:0/2341899414 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f39e000cd50 con 0x7f39f012f770 2024-09-21T01:46:16.619 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.617+0000 7f39e67fc640 1 -- 172.21.15.105:0/2341899414 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f39e0004020 con 0x7f39f012f770 2024-09-21T01:46:16.619 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.617+0000 7f39e67fc640 1 -- 172.21.15.105:0/2341899414 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f39e0004450 con 0x7f39f012f770 2024-09-21T01:46:16.619 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.617+0000 7f39f7bf5640 1 -- 172.21.15.105:0/2341899414 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f39f0129d50 con 0x7f39f012f770 2024-09-21T01:46:16.619 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.617+0000 7f39f7bf5640 1 -- 172.21.15.105:0/2341899414 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f39f012b570 con 0x7f39f012f770 2024-09-21T01:46:16.619 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.618+0000 7f39f7bf5640 1 -- 172.21.15.105:0/2341899414 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f39f0070c50 con 0x7f39f012f770 2024-09-21T01:46:16.619 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.618+0000 7f39e67fc640 1 -- 172.21.15.105:0/2341899414 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 9) ==== 63442+0+0 (secure 0 0 0) 0x7f39e00035f0 con 0x7f39f012f770 2024-09-21T01:46:16.620 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.619+0000 7f39e67fc640 1 -- 172.21.15.105:0/2341899414 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7f39e0069860 con 0x7f39f012f770 2024-09-21T01:46:16.622 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.622+0000 7f39e67fc640 1 -- 172.21.15.105:0/2341899414 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f39f0129d50 con 0x7f39f012f770 2024-09-21T01:46:16.677 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.677+0000 7f8f023eb640 1 -- 172.21.15.105:0/774557556 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7f8efc072b30 con 0x7f8efc12d4a0 2024-09-21T01:46:16.677 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.677+0000 7f8ef8ff9640 1 -- 172.21.15.105:0/774557556 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f8efc072b30 con 0x7f8efc12d4a0 2024-09-21T01:46:16.678 INFO:teuthology.orchestra.run.smithi105.stdout:38654705685 2024-09-21T01:46:16.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.679+0000 7f8f023eb640 1 -- 172.21.15.105:0/774557556 >> v1:172.21.15.105:6816/2838005468 conn(0x7f8ee8051450 legacy=0x7f8ee8053960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:16.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.679+0000 7f8f023eb640 1 -- 172.21.15.105:0/774557556 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8efc12d4a0 msgr2=0x7f8efc1c3880 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:16.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.679+0000 7f8f023eb640 1 --2- 172.21.15.105:0/774557556 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8efc12d4a0 0x7f8efc1c3880 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f8ef400eb50 tx=0x7f8ef400c6a0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.679+0000 7f8f023eb640 1 -- 172.21.15.105:0/774557556 shutdown_connections 2024-09-21T01:46:16.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.679+0000 7f8f023eb640 1 --2- 172.21.15.105:0/774557556 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8efc12d4a0 0x7f8efc1c3880 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.679+0000 7f8f023eb640 1 --2- 172.21.15.105:0/774557556 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8efc12a400 0x7f8efc132220 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.679+0000 7f8f023eb640 1 --2- 172.21.15.105:0/774557556 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8efc073960 0x7f8efc131ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.679+0000 7f8f023eb640 1 -- 172.21.15.105:0/774557556 >> 172.21.15.105:0/774557556 conn(0x7f8efc09fc30 msgr2=0x7f8efc12cc70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:16.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.679+0000 7f8f023eb640 1 -- 172.21.15.105:0/774557556 shutdown_connections 2024-09-21T01:46:16.680 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.679+0000 7f8f023eb640 1 -- 172.21.15.105:0/774557556 wait complete. 2024-09-21T01:46:16.691 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705684 got 38654705685 for osd.0 2024-09-21T01:46:16.691 DEBUG:teuthology.parallel:result is None 2024-09-21T01:46:16.722 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.722+0000 7fd4697eb640 1 -- 172.21.15.105:0/3462924711 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7fd46406a690 con 0x7fd4640990c0 2024-09-21T01:46:16.722 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.722+0000 7fd44b7fe640 1 -- 172.21.15.105:0/3462924711 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7fd458068560 con 0x7fd4640990c0 2024-09-21T01:46:16.722 INFO:teuthology.orchestra.run.smithi105.stdout:38654705684 2024-09-21T01:46:16.725 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.724+0000 7fd4697eb640 1 -- 172.21.15.105:0/3462924711 >> v1:172.21.15.105:6816/2838005468 conn(0x7fd450051420 legacy=0x7fd450053930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:16.725 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.724+0000 7fd4697eb640 1 -- 172.21.15.105:0/3462924711 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd4640990c0 msgr2=0x7fd464070030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:16.725 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.724+0000 7fd4697eb640 1 --2- 172.21.15.105:0/3462924711 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd4640990c0 0x7fd464070030 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 crypto rx=0x7fd45800e9f0 tx=0x7fd45800eeb0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.725 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.724+0000 7fd4697eb640 1 -- 172.21.15.105:0/3462924711 shutdown_connections 2024-09-21T01:46:16.725 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.724+0000 7fd4697eb640 1 --2- 172.21.15.105:0/3462924711 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd46412ba20 0x7fd464070570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.725 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.724+0000 7fd4697eb640 1 --2- 172.21.15.105:0/3462924711 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd4640990c0 0x7fd464070030 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.725 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.724+0000 7fd4697eb640 1 --2- 172.21.15.105:0/3462924711 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd464073dd0 0x7fd46406faf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.725 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.724+0000 7fd4697eb640 1 -- 172.21.15.105:0/3462924711 >> 172.21.15.105:0/3462924711 conn(0x7fd4640a0160 msgr2=0x7fd46412dcd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:16.725 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.724+0000 7fd4697eb640 1 -- 172.21.15.105:0/3462924711 shutdown_connections 2024-09-21T01:46:16.725 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.724+0000 7fd4697eb640 1 -- 172.21.15.105:0/3462924711 wait complete. 2024-09-21T01:46:16.735 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705684 for osd.1 2024-09-21T01:46:16.735 DEBUG:teuthology.parallel:result is None 2024-09-21T01:46:16.779 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.778+0000 7f39f7bf5640 1 -- 172.21.15.105:0/2341899414 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7f39f006a780 con 0x7f39f012f770 2024-09-21T01:46:16.779 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.779+0000 7f39e67fc640 1 -- 172.21.15.105:0/2341899414 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+12 (secure 0 0 0) 0x7f39e00093f0 con 0x7f39f012f770 2024-09-21T01:46:16.779 INFO:teuthology.orchestra.run.smithi105.stdout:38654705684 2024-09-21T01:46:16.781 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.781+0000 7f39f7bf5640 1 -- 172.21.15.105:0/2341899414 >> v1:172.21.15.105:6816/2838005468 conn(0x7f39d4051450 legacy=0x7f39d4053960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:16.781 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.781+0000 7f39f7bf5640 1 -- 172.21.15.105:0/2341899414 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f39f012f770 msgr2=0x7f39f01c6030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:16.781 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.781+0000 7f39f7bf5640 1 --2- 172.21.15.105:0/2341899414 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f39f012f770 0x7f39f01c6030 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f39e000ea20 tx=0x7f39e000eee0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.782 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.781+0000 7f39f7bf5640 1 -- 172.21.15.105:0/2341899414 shutdown_connections 2024-09-21T01:46:16.782 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.781+0000 7f39f7bf5640 1 --2- 172.21.15.105:0/2341899414 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f39f0134e80 0x7f39f01c6570 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.782 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.781+0000 7f39f7bf5640 1 --2- 172.21.15.105:0/2341899414 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f39f012f770 0x7f39f01c6030 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.782 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.781+0000 7f39f7bf5640 1 --2- 172.21.15.105:0/2341899414 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f39f0071730 0x7f39f01c5af0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.782 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.781+0000 7f39f7bf5640 1 -- 172.21.15.105:0/2341899414 >> 172.21.15.105:0/2341899414 conn(0x7f39f00a2080 msgr2=0x7f39f0126d20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:16.782 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.781+0000 7f39f7bf5640 1 -- 172.21.15.105:0/2341899414 shutdown_connections 2024-09-21T01:46:16.783 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.781+0000 7f39f7bf5640 1 -- 172.21.15.105:0/2341899414 wait complete. 2024-09-21T01:46:16.792 INFO:tasks.ceph.ceph_manager.ceph:need seq 38654705683 got 38654705684 for osd.2 2024-09-21T01:46:16.792 DEBUG:teuthology.parallel:result is None 2024-09-21T01:46:16.792 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean 2024-09-21T01:46:16.792 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T01:46:16.902 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.902+0000 7f0a4ffa8640 1 -- 172.21.15.105:0/898714776 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0a48073b30 msgr2=0x7f0a4812d910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:16.902 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.902+0000 7f0a4ffa8640 1 --2- 172.21.15.105:0/898714776 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0a48073b30 0x7f0a4812d910 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f0a44009830 tx=0x7f0a4402a3e0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.902+0000 7f0a4ffa8640 1 -- 172.21.15.105:0/898714776 shutdown_connections 2024-09-21T01:46:16.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.902+0000 7f0a4ffa8640 1 --2- 172.21.15.105:0/898714776 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0a4812a400 0x7f0a4812a810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.902+0000 7f0a4ffa8640 1 --2- 172.21.15.105:0/898714776 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0a4812de50 0x7f0a48129ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.902+0000 7f0a4ffa8640 1 --2- 172.21.15.105:0/898714776 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0a48073b30 0x7f0a4812d910 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.902+0000 7f0a4ffa8640 1 -- 172.21.15.105:0/898714776 >> 172.21.15.105:0/898714776 conn(0x7f0a4809fe70 msgr2=0x7f0a480a0280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:16.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.902+0000 7f0a4ffa8640 1 -- 172.21.15.105:0/898714776 shutdown_connections 2024-09-21T01:46:16.903 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.902+0000 7f0a4ffa8640 1 -- 172.21.15.105:0/898714776 wait complete. 2024-09-21T01:46:16.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.903+0000 7f0a4ffa8640 1 Processor -- start 2024-09-21T01:46:16.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.903+0000 7f0a4ffa8640 1 -- start start 2024-09-21T01:46:16.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.903+0000 7f0a4ffa8640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0a48073b30 0x7f0a480989a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:16.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.903+0000 7f0a4ffa8640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0a4812a400 0x7f0a48098ee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:16.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.903+0000 7f0a4ffa8640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0a4812de50 0x7f0a48099420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:16.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.903+0000 7f0a4ffa8640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f0a48135270 con 0x7f0a4812de50 2024-09-21T01:46:16.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.903+0000 7f0a4ffa8640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f0a481350f0 con 0x7f0a4812a400 2024-09-21T01:46:16.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.903+0000 7f0a4ffa8640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f0a481353f0 con 0x7f0a48073b30 2024-09-21T01:46:16.905 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.903+0000 7f0a4dd44640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0a48073b30 0x7f0a480989a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:16.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.903+0000 7f0a4e545640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0a4812de50 0x7f0a48099420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:16.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.903+0000 7f0a4d543640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0a4812a400 0x7f0a48098ee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:16.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.903+0000 7f0a4dd44640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0a48073b30 0x7f0a480989a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:50304/0 (socket says 172.21.15.105:50304) 2024-09-21T01:46:16.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.903+0000 7f0a4e545640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0a4812de50 0x7f0a48099420 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:50728/0 (socket says 172.21.15.105:50728) 2024-09-21T01:46:16.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.903+0000 7f0a4dd44640 1 -- 172.21.15.105:0/649490553 learned_addr learned my addr 172.21.15.105:0/649490553 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:16.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.904+0000 7f0a4dd44640 1 -- 172.21.15.105:0/649490553 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0a4812a400 msgr2=0x7f0a48098ee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:16.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.904+0000 7f0a4dd44640 1 --2- 172.21.15.105:0/649490553 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0a4812a400 0x7f0a48098ee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.904+0000 7f0a4dd44640 1 -- 172.21.15.105:0/649490553 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0a4812de50 msgr2=0x7f0a48099420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:16.906 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.904+0000 7f0a4dd44640 1 --2- 172.21.15.105:0/649490553 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0a4812de50 0x7f0a48099420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:16.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.904+0000 7f0a4dd44640 1 -- 172.21.15.105:0/649490553 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0a48133140 con 0x7f0a48073b30 2024-09-21T01:46:16.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.904+0000 7f0a4e545640 1 --2- 172.21.15.105:0/649490553 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0a4812de50 0x7f0a48099420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:16.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.904+0000 7f0a4dd44640 1 --2- 172.21.15.105:0/649490553 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0a48073b30 0x7f0a480989a0 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f0a44000c00 tx=0x7f0a44033c20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:16.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.904+0000 7f0a3e7fc640 1 -- 172.21.15.105:0/649490553 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0a44005c90 con 0x7f0a48073b30 2024-09-21T01:46:16.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.904+0000 7f0a3e7fc640 1 -- 172.21.15.105:0/649490553 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f0a44031e50 con 0x7f0a48073b30 2024-09-21T01:46:16.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.904+0000 7f0a3e7fc640 1 -- 172.21.15.105:0/649490553 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0a44033290 con 0x7f0a48073b30 2024-09-21T01:46:16.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.904+0000 7f0a4ffa8640 1 -- 172.21.15.105:0/649490553 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0a481341f0 con 0x7f0a48073b30 2024-09-21T01:46:16.907 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.904+0000 7f0a4ffa8640 1 -- 172.21.15.105:0/649490553 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f0a4808c210 con 0x7f0a48073b30 2024-09-21T01:46:16.908 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.905+0000 7f0a4ffa8640 1 -- 172.21.15.105:0/649490553 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0a4812a810 con 0x7f0a48073b30 2024-09-21T01:46:16.908 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.905+0000 7f0a3e7fc640 1 -- 172.21.15.105:0/649490553 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 9) ==== 63442+0+0 (secure 0 0 0) 0x7f0a44033430 con 0x7f0a48073b30 2024-09-21T01:46:16.908 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.906+0000 7f0a3e7fc640 1 -- 172.21.15.105:0/649490553 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7f0a4408a120 con 0x7f0a48073b30 2024-09-21T01:46:16.910 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:16.910+0000 7f0a3e7fc640 1 -- 172.21.15.105:0/649490553 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f0a48133140 con 0x7f0a48073b30 2024-09-21T01:46:17.058 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.058+0000 7f0a4ffa8640 1 -- 172.21.15.105:0/649490553 --> v1:172.21.15.105:6816/2838005468 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f0a4806a740 con 0x7f0a2c051450 2024-09-21T01:46:17.059 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.058+0000 7f0a3e7fc640 1 -- 172.21.15.105:0/649490553 <== mgr.14125 v1:172.21.15.105:6816/2838005468 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32018 (unknown 2965378022 0 749719929) 0x7f0a4806a740 con 0x7f0a2c051450 2024-09-21T01:46:17.059 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:17.059 INFO:teuthology.orchestra.run.smithi105.stderr:dumped all 2024-09-21T01:46:17.061 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.060+0000 7f0a4ffa8640 1 -- 172.21.15.105:0/649490553 >> v1:172.21.15.105:6816/2838005468 conn(0x7f0a2c051450 legacy=0x7f0a2c053960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:17.061 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.060+0000 7f0a4ffa8640 1 -- 172.21.15.105:0/649490553 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0a48073b30 msgr2=0x7f0a480989a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:17.061 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.060+0000 7f0a4ffa8640 1 --2- 172.21.15.105:0/649490553 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0a48073b30 0x7f0a480989a0 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f0a44000c00 tx=0x7f0a44033c20 comp rx=0 tx=0).stop 2024-09-21T01:46:17.061 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.061+0000 7f0a4ffa8640 1 -- 172.21.15.105:0/649490553 shutdown_connections 2024-09-21T01:46:17.062 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.061+0000 7f0a4ffa8640 1 --2- 172.21.15.105:0/649490553 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0a4812de50 0x7f0a48099420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.062 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.061+0000 7f0a4ffa8640 1 --2- 172.21.15.105:0/649490553 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0a4812a400 0x7f0a48098ee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.062 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.061+0000 7f0a4ffa8640 1 --2- 172.21.15.105:0/649490553 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0a48073b30 0x7f0a480989a0 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.062 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.061+0000 7f0a4ffa8640 1 -- 172.21.15.105:0/649490553 >> 172.21.15.105:0/649490553 conn(0x7f0a4809fe70 msgr2=0x7f0a48127320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:17.062 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.061+0000 7f0a4ffa8640 1 -- 172.21.15.105:0/649490553 shutdown_connections 2024-09-21T01:46:17.062 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.061+0000 7f0a4ffa8640 1 -- 172.21.15.105:0/649490553 wait complete. 2024-09-21T01:46:17.072 INFO:teuthology.orchestra.run.smithi105.stdout:{"pg_ready":true,"pg_map":{"version":4,"stamp":"2024-09-21T01:46:15.497719+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":109916,"kb_used_data":1832,"kb_used_omap":6,"kb_used_meta":108025,"kb_avail":377377444,"statfs":{"total":386547056640,"available":386434502656,"internally_reserved":0,"allocated":1875968,"data_stored":1380376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6374,"internal_metadata":110618394},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.999495"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T01:45:03.552881+0000","last_change":"2024-09-21T01:44:57.534395+0000","last_active":"2024-09-21T01:45:03.552881+0000","last_peered":"2024-09-21T01:45:03.552881+0000","last_clean":"2024-09-21T01:45:03.552881+0000","last_became_active":"2024-09-21T01:44:55.526300+0000","last_became_peered":"2024-09-21T01:44:55.526300+0000","last_unstale":"2024-09-21T01:45:03.552881+0000","last_undegraded":"2024-09-21T01:45:03.552881+0000","last_fullsized":"2024-09-21T01:45:03.552881+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:38.742537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040868700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T01:45:03.595885+0000","last_change":"2024-09-21T01:44:57.531141+0000","last_active":"2024-09-21T01:45:03.595885+0000","last_peered":"2024-09-21T01:45:03.595885+0000","last_clean":"2024-09-21T01:45:03.595885+0000","last_became_active":"2024-09-21T01:44:55.527636+0000","last_became_peered":"2024-09-21T01:44:55.527636+0000","last_unstale":"2024-09-21T01:45:03.595885+0000","last_undegraded":"2024-09-21T01:45:03.595885+0000","last_fullsized":"2024-09-21T01:45:03.595885+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:57:35.618887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055542700000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T01:45:03.552983+0000","last_change":"2024-09-21T01:44:57.535152+0000","last_active":"2024-09-21T01:45:03.552983+0000","last_peered":"2024-09-21T01:45:03.552983+0000","last_clean":"2024-09-21T01:45:03.552983+0000","last_became_active":"2024-09-21T01:44:55.526163+0000","last_became_peered":"2024-09-21T01:44:55.526163+0000","last_unstale":"2024-09-21T01:45:03.552983+0000","last_undegraded":"2024-09-21T01:45:03.552983+0000","last_fullsized":"2024-09-21T01:45:03.552983+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:57:35.618887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00119484,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T01:45:03.595882+0000","last_change":"2024-09-21T01:44:57.531202+0000","last_active":"2024-09-21T01:45:03.595882+0000","last_peered":"2024-09-21T01:45:03.595882+0000","last_clean":"2024-09-21T01:45:03.595882+0000","last_became_active":"2024-09-21T01:44:55.526110+0000","last_became_peered":"2024-09-21T01:44:55.526110+0000","last_unstale":"2024-09-21T01:45:03.595882+0000","last_undegraded":"2024-09-21T01:45:03.595882+0000","last_fullsized":"2024-09-21T01:45:03.595882+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:38.742537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063215099999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":21,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T01:45:03.552860+0000","last_change":"2024-09-21T01:44:57.535093+0000","last_active":"2024-09-21T01:45:03.552860+0000","last_peered":"2024-09-21T01:45:03.552860+0000","last_clean":"2024-09-21T01:45:03.552860+0000","last_became_active":"2024-09-21T01:44:55.529008+0000","last_became_peered":"2024-09-21T01:44:55.529008+0000","last_unstale":"2024-09-21T01:45:03.552860+0000","last_undegraded":"2024-09-21T01:45:03.552860+0000","last_fullsized":"2024-09-21T01:45:03.552860+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:43.118995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010245149999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T01:45:03.552808+0000","last_change":"2024-09-21T01:44:57.531535+0000","last_active":"2024-09-21T01:45:03.552808+0000","last_peered":"2024-09-21T01:45:03.552808+0000","last_clean":"2024-09-21T01:45:03.552808+0000","last_became_active":"2024-09-21T01:44:55.526110+0000","last_became_peered":"2024-09-21T01:44:55.526110+0000","last_unstale":"2024-09-21T01:45:03.552808+0000","last_undegraded":"2024-09-21T01:45:03.552808+0000","last_fullsized":"2024-09-21T01:45:03.552808+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T05:54:08.666566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080666500000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T01:45:03.552969+0000","last_change":"2024-09-21T01:44:57.534522+0000","last_active":"2024-09-21T01:45:03.552969+0000","last_peered":"2024-09-21T01:45:03.552969+0000","last_clean":"2024-09-21T01:45:03.552969+0000","last_became_active":"2024-09-21T01:44:55.528489+0000","last_became_peered":"2024-09-21T01:44:55.528489+0000","last_unstale":"2024-09-21T01:45:03.552969+0000","last_undegraded":"2024-09-21T01:45:03.552969+0000","last_fullsized":"2024-09-21T01:45:03.552969+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T07:40:38.634948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058759600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":20,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T01:45:03.552979+0000","last_change":"2024-09-21T01:44:57.534320+0000","last_active":"2024-09-21T01:45:03.552979+0000","last_peered":"2024-09-21T01:45:03.552979+0000","last_clean":"2024-09-21T01:45:03.552979+0000","last_became_active":"2024-09-21T01:44:55.525577+0000","last_became_peered":"2024-09-21T01:44:55.525577+0000","last_unstale":"2024-09-21T01:45:03.552979+0000","last_undegraded":"2024-09-21T01:45:03.552979+0000","last_fullsized":"2024-09-21T01:45:03.552979+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T03:49:16.598027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036256899999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"12'97","reported_seq":144,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T01:45:03.595933+0000","last_change":"2024-09-21T01:44:51.556547+0000","last_active":"2024-09-21T01:45:03.595933+0000","last_peered":"2024-09-21T01:45:03.595933+0000","last_clean":"2024-09-21T01:45:03.595933+0000","last_became_active":"2024-09-21T01:44:51.555535+0000","last_became_peered":"2024-09-21T01:44:51.555535+0000","last_unstale":"2024-09-21T01:45:03.595933+0000","last_undegraded":"2024-09-21T01:45:03.595933+0000","last_fullsized":"2024-09-21T01:45:03.595933+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:50.489745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:50.489745+0000","last_clean_scrub_stamp":"2024-09-21T01:44:50.489745+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T03:15:49.853900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":9,"seq":38654705684,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27192,"kb_used_data":172,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344648,"statfs":{"total":96636764160,"available":96608919552,"internally_reserved":0,"allocated":176128,"data_stored":50780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":9,"seq":38654705684,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27188,"kb_used_data":168,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344652,"statfs":{"total":96636764160,"available":96608923648,"internally_reserved":0,"allocated":172032,"data_stored":50761,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705684,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27772,"kb_used_data":752,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344068,"statfs":{"total":96636764160,"available":96608325632,"internally_reserved":0,"allocated":770048,"data_stored":641148,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705685,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27764,"kb_used_data":740,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344076,"statfs":{"total":96636764160,"available":96608333824,"internally_reserved":0,"allocated":757760,"data_stored":637687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T01:46:17.073 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T01:46:17.184 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.183+0000 7f567aa20640 1 -- 172.21.15.105:0/2709494404 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f567412c440 msgr2=0x7f5674132e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:17.184 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.183+0000 7f567aa20640 1 --2- 172.21.15.105:0/2709494404 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f567412c440 0x7f5674132e60 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7f566c005fd0 tx=0x7f566c02a970 comp rx=0 tx=0).stop 2024-09-21T01:46:17.184 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.183+0000 7f567aa20640 1 -- 172.21.15.105:0/2709494404 shutdown_connections 2024-09-21T01:46:17.184 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.183+0000 7f567aa20640 1 --2- 172.21.15.105:0/2709494404 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f567412c440 0x7f5674132e60 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.184 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.183+0000 7f567aa20640 1 --2- 172.21.15.105:0/2709494404 >> v2:172.21.15.105:3300/0 conn(0x7f567412baf0 0x7f567412bf00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.184 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.183+0000 7f567aa20640 1 --2- 172.21.15.105:0/2709494404 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f56740733e0 0x7f56740737f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.184 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.183+0000 7f567aa20640 1 -- 172.21.15.105:0/2709494404 >> 172.21.15.105:0/2709494404 conn(0x7f567409b7c0 msgr2=0x7f567409bbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:17.185 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.183+0000 7f567aa20640 1 -- 172.21.15.105:0/2709494404 shutdown_connections 2024-09-21T01:46:17.185 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.183+0000 7f567aa20640 1 -- 172.21.15.105:0/2709494404 wait complete. 2024-09-21T01:46:17.187 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.184+0000 7f567aa20640 1 Processor -- start 2024-09-21T01:46:17.187 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.184+0000 7f567aa20640 1 -- start start 2024-09-21T01:46:17.187 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.185+0000 7f567aa20640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f56740733e0 0x7f567412cb00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:17.187 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.185+0000 7f567aa20640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f567412baf0 0x7f567412d040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:17.187 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.185+0000 7f567aa20640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f567412c440 0x7f567412d580 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:17.187 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.185+0000 7f567aa20640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f5674133920 con 0x7f567412baf0 2024-09-21T01:46:17.187 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.185+0000 7f567aa20640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f56741337a0 con 0x7f56740733e0 2024-09-21T01:46:17.187 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.185+0000 7f567aa20640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f5674133aa0 con 0x7f567412c440 2024-09-21T01:46:17.187 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.185+0000 7f5673fff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f56740733e0 0x7f567412cb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:17.187 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.185+0000 7f56737fe640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f567412baf0 0x7f567412d040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:17.187 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.185+0000 7f5673fff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f56740733e0 0x7f567412cb00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:60132/0 (socket says 172.21.15.105:60132) 2024-09-21T01:46:17.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.185+0000 7f5673fff640 1 -- 172.21.15.105:0/3358677964 learned_addr learned my addr 172.21.15.105:0/3358677964 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:17.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.185+0000 7f5678fbd640 1 --2- 172.21.15.105:0/3358677964 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f567412c440 0x7f567412d580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:17.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.185+0000 7f5673fff640 1 -- 172.21.15.105:0/3358677964 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f567412c440 msgr2=0x7f567412d580 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:17.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.185+0000 7f5673fff640 1 --2- 172.21.15.105:0/3358677964 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f567412c440 0x7f567412d580 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.185+0000 7f5673fff640 1 -- 172.21.15.105:0/3358677964 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f567412baf0 msgr2=0x7f567412d040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:17.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.185+0000 7f5673fff640 1 --2- 172.21.15.105:0/3358677964 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f567412baf0 0x7f567412d040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.185+0000 7f5673fff640 1 -- 172.21.15.105:0/3358677964 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f567408cc20 con 0x7f56740733e0 2024-09-21T01:46:17.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.185+0000 7f56737fe640 1 --2- 172.21.15.105:0/3358677964 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f567412baf0 0x7f567412d040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:17.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.185+0000 7f5673fff640 1 --2- 172.21.15.105:0/3358677964 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f56740733e0 0x7f567412cb00 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f566400e9f0 tx=0x7f566400eeb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:17.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.185+0000 7f5670ff9640 1 -- 172.21.15.105:0/3358677964 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f566400cd80 con 0x7f56740733e0 2024-09-21T01:46:17.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.185+0000 7f5670ff9640 1 -- 172.21.15.105:0/3358677964 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f5664004020 con 0x7f56740733e0 2024-09-21T01:46:17.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.185+0000 7f567aa20640 1 -- 172.21.15.105:0/3358677964 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f567408c630 con 0x7f56740733e0 2024-09-21T01:46:17.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.185+0000 7f5670ff9640 1 -- 172.21.15.105:0/3358677964 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5664004470 con 0x7f56740733e0 2024-09-21T01:46:17.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.186+0000 7f567aa20640 1 -- 172.21.15.105:0/3358677964 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f56741cbf70 con 0x7f56740733e0 2024-09-21T01:46:17.188 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.187+0000 7f567aa20640 1 -- 172.21.15.105:0/3358677964 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5674069510 con 0x7f56740733e0 2024-09-21T01:46:17.189 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.187+0000 7f5670ff9640 1 -- 172.21.15.105:0/3358677964 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 9) ==== 63442+0+0 (secure 0 0 0) 0x7f56640035f0 con 0x7f56740733e0 2024-09-21T01:46:17.189 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.187+0000 7f5670ff9640 1 -- 172.21.15.105:0/3358677964 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7f5664069800 con 0x7f56740733e0 2024-09-21T01:46:17.191 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.191+0000 7f5670ff9640 1 -- 172.21.15.105:0/3358677964 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f5674069510 con 0x7f56740733e0 2024-09-21T01:46:17.334 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.334+0000 7f567aa20640 1 -- 172.21.15.105:0/3358677964 --> v1:172.21.15.105:6816/2838005468 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f5674134060 con 0x7f5660051420 2024-09-21T01:46:17.335 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.335+0000 7f5670ff9640 1 -- 172.21.15.105:0/3358677964 <== mgr.14125 v1:172.21.15.105:6816/2838005468 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32018 (unknown 2965378022 0 749719929) 0x7f5674134060 con 0x7f5660051420 2024-09-21T01:46:17.335 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:17.336 INFO:teuthology.orchestra.run.smithi105.stderr:dumped all 2024-09-21T01:46:17.337 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.337+0000 7f567aa20640 1 -- 172.21.15.105:0/3358677964 >> v1:172.21.15.105:6816/2838005468 conn(0x7f5660051420 legacy=0x7f5660053930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:17.338 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.337+0000 7f567aa20640 1 -- 172.21.15.105:0/3358677964 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f56740733e0 msgr2=0x7f567412cb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:17.338 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.337+0000 7f567aa20640 1 --2- 172.21.15.105:0/3358677964 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f56740733e0 0x7f567412cb00 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 crypto rx=0x7f566400e9f0 tx=0x7f566400eeb0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.338 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.337+0000 7f567aa20640 1 -- 172.21.15.105:0/3358677964 shutdown_connections 2024-09-21T01:46:17.338 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.337+0000 7f567aa20640 1 --2- 172.21.15.105:0/3358677964 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f567412c440 0x7f567412d580 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.338 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.337+0000 7f567aa20640 1 --2- 172.21.15.105:0/3358677964 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f567412baf0 0x7f567412d040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.338 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.337+0000 7f567aa20640 1 --2- 172.21.15.105:0/3358677964 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f56740733e0 0x7f567412cb00 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.338 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.337+0000 7f567aa20640 1 -- 172.21.15.105:0/3358677964 >> 172.21.15.105:0/3358677964 conn(0x7f567409b7c0 msgr2=0x7f5674098e70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:17.338 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.337+0000 7f567aa20640 1 -- 172.21.15.105:0/3358677964 shutdown_connections 2024-09-21T01:46:17.338 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.337+0000 7f567aa20640 1 -- 172.21.15.105:0/3358677964 wait complete. 2024-09-21T01:46:17.348 INFO:teuthology.orchestra.run.smithi105.stdout:{"pg_ready":true,"pg_map":{"version":4,"stamp":"2024-09-21T01:46:15.497719+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":139,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100,"ondisk_log_size":100,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":109916,"kb_used_data":1832,"kb_used_omap":6,"kb_used_meta":108025,"kb_avail":377377444,"statfs":{"total":386547056640,"available":386434502656,"internally_reserved":0,"allocated":1875968,"data_stored":1380376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6374,"internal_metadata":110618394},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.999495"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T01:45:03.552881+0000","last_change":"2024-09-21T01:44:57.534395+0000","last_active":"2024-09-21T01:45:03.552881+0000","last_peered":"2024-09-21T01:45:03.552881+0000","last_clean":"2024-09-21T01:45:03.552881+0000","last_became_active":"2024-09-21T01:44:55.526300+0000","last_became_peered":"2024-09-21T01:44:55.526300+0000","last_unstale":"2024-09-21T01:45:03.552881+0000","last_undegraded":"2024-09-21T01:45:03.552881+0000","last_fullsized":"2024-09-21T01:45:03.552881+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:38.742537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040868700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T01:45:03.595885+0000","last_change":"2024-09-21T01:44:57.531141+0000","last_active":"2024-09-21T01:45:03.595885+0000","last_peered":"2024-09-21T01:45:03.595885+0000","last_clean":"2024-09-21T01:45:03.595885+0000","last_became_active":"2024-09-21T01:44:55.527636+0000","last_became_peered":"2024-09-21T01:44:55.527636+0000","last_unstale":"2024-09-21T01:45:03.595885+0000","last_undegraded":"2024-09-21T01:45:03.595885+0000","last_fullsized":"2024-09-21T01:45:03.595885+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:57:35.618887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055542700000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T01:45:03.552983+0000","last_change":"2024-09-21T01:44:57.535152+0000","last_active":"2024-09-21T01:45:03.552983+0000","last_peered":"2024-09-21T01:45:03.552983+0000","last_clean":"2024-09-21T01:45:03.552983+0000","last_became_active":"2024-09-21T01:44:55.526163+0000","last_became_peered":"2024-09-21T01:44:55.526163+0000","last_unstale":"2024-09-21T01:45:03.552983+0000","last_undegraded":"2024-09-21T01:45:03.552983+0000","last_fullsized":"2024-09-21T01:45:03.552983+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:57:35.618887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00119484,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T01:45:03.595882+0000","last_change":"2024-09-21T01:44:57.531202+0000","last_active":"2024-09-21T01:45:03.595882+0000","last_peered":"2024-09-21T01:45:03.595882+0000","last_clean":"2024-09-21T01:45:03.595882+0000","last_became_active":"2024-09-21T01:44:55.526110+0000","last_became_peered":"2024-09-21T01:44:55.526110+0000","last_unstale":"2024-09-21T01:45:03.595882+0000","last_undegraded":"2024-09-21T01:45:03.595882+0000","last_fullsized":"2024-09-21T01:45:03.595882+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:51:38.742537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063215099999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":21,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T01:45:03.552860+0000","last_change":"2024-09-21T01:44:57.535093+0000","last_active":"2024-09-21T01:45:03.552860+0000","last_peered":"2024-09-21T01:45:03.552860+0000","last_clean":"2024-09-21T01:45:03.552860+0000","last_became_active":"2024-09-21T01:44:55.529008+0000","last_became_peered":"2024-09-21T01:44:55.529008+0000","last_unstale":"2024-09-21T01:45:03.552860+0000","last_undegraded":"2024-09-21T01:45:03.552860+0000","last_fullsized":"2024-09-21T01:45:03.552860+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:43.118995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010245149999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T01:45:03.552808+0000","last_change":"2024-09-21T01:44:57.531535+0000","last_active":"2024-09-21T01:45:03.552808+0000","last_peered":"2024-09-21T01:45:03.552808+0000","last_clean":"2024-09-21T01:45:03.552808+0000","last_became_active":"2024-09-21T01:44:55.526110+0000","last_became_peered":"2024-09-21T01:44:55.526110+0000","last_unstale":"2024-09-21T01:45:03.552808+0000","last_undegraded":"2024-09-21T01:45:03.552808+0000","last_fullsized":"2024-09-21T01:45:03.552808+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T05:54:08.666566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080666500000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T01:45:03.552969+0000","last_change":"2024-09-21T01:44:57.534522+0000","last_active":"2024-09-21T01:45:03.552969+0000","last_peered":"2024-09-21T01:45:03.552969+0000","last_clean":"2024-09-21T01:45:03.552969+0000","last_became_active":"2024-09-21T01:44:55.528489+0000","last_became_peered":"2024-09-21T01:44:55.528489+0000","last_unstale":"2024-09-21T01:45:03.552969+0000","last_undegraded":"2024-09-21T01:45:03.552969+0000","last_fullsized":"2024-09-21T01:45:03.552969+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T07:40:38.634948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058759600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":20,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T01:45:03.552979+0000","last_change":"2024-09-21T01:44:57.534320+0000","last_active":"2024-09-21T01:45:03.552979+0000","last_peered":"2024-09-21T01:45:03.552979+0000","last_clean":"2024-09-21T01:45:03.552979+0000","last_became_active":"2024-09-21T01:44:55.525577+0000","last_became_peered":"2024-09-21T01:44:55.525577+0000","last_unstale":"2024-09-21T01:45:03.552979+0000","last_undegraded":"2024-09-21T01:45:03.552979+0000","last_fullsized":"2024-09-21T01:45:03.552979+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T03:49:16.598027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036256899999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"12'97","reported_seq":144,"reported_epoch":19,"state":"active+clean","last_fresh":"2024-09-21T01:45:03.595933+0000","last_change":"2024-09-21T01:44:51.556547+0000","last_active":"2024-09-21T01:45:03.595933+0000","last_peered":"2024-09-21T01:45:03.595933+0000","last_clean":"2024-09-21T01:45:03.595933+0000","last_became_active":"2024-09-21T01:44:51.555535+0000","last_became_peered":"2024-09-21T01:44:51.555535+0000","last_unstale":"2024-09-21T01:45:03.595933+0000","last_undegraded":"2024-09-21T01:45:03.595933+0000","last_fullsized":"2024-09-21T01:45:03.595933+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:50.489745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:50.489745+0000","last_clean_scrub_stamp":"2024-09-21T01:44:50.489745+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T03:15:49.853900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":9,"seq":38654705684,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27192,"kb_used_data":172,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344648,"statfs":{"total":96636764160,"available":96608919552,"internally_reserved":0,"allocated":176128,"data_stored":50780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":9,"seq":38654705684,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27188,"kb_used_data":168,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344652,"statfs":{"total":96636764160,"available":96608923648,"internally_reserved":0,"allocated":172032,"data_stored":50761,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705684,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27772,"kb_used_data":752,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344068,"statfs":{"total":96636764160,"available":96608325632,"internally_reserved":0,"allocated":770048,"data_stored":641148,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705685,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27764,"kb_used_data":740,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":94344076,"statfs":{"total":96636764160,"available":96608333824,"internally_reserved":0,"allocated":757760,"data_stored":637687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T01:46:17.349 INFO:tasks.ceph.ceph_manager.ceph:clean! 2024-09-21T01:46:17.349 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T01:46:17.349 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy 2024-09-21T01:46:17.350 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph health --format=json 2024-09-21T01:46:17.458 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.458+0000 7fc2b1534640 1 -- 172.21.15.105:0/363761750 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fc2ac071730 msgr2=0x7fc2ac134940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:17.459 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.458+0000 7fc2b1534640 1 --2- 172.21.15.105:0/363761750 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fc2ac071730 0x7fc2ac134940 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7fc2a0009860 tx=0x7fc2a002a460 comp rx=0 tx=0).stop 2024-09-21T01:46:17.459 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.458+0000 7fc2b1534640 1 -- 172.21.15.105:0/363761750 shutdown_connections 2024-09-21T01:46:17.459 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.458+0000 7fc2b1534640 1 --2- 172.21.15.105:0/363761750 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fc2ac134e80 0x7fc2ac1372a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.459 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.458+0000 7fc2b1534640 1 --2- 172.21.15.105:0/363761750 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fc2ac071730 0x7fc2ac134940 unknown :-1 s=CLOSED pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.459 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.458+0000 7fc2b1534640 1 -- 172.21.15.105:0/363761750 >> 172.21.15.105:0/363761750 conn(0x7fc2ac0a2080 msgr2=0x7fc2ac0a2490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:17.459 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.458+0000 7fc2b1534640 1 -- 172.21.15.105:0/363761750 shutdown_connections 2024-09-21T01:46:17.459 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.458+0000 7fc2b1534640 1 -- 172.21.15.105:0/363761750 wait complete. 2024-09-21T01:46:17.461 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.459+0000 7fc2b1534640 1 Processor -- start 2024-09-21T01:46:17.461 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.459+0000 7fc2b1534640 1 -- start start 2024-09-21T01:46:17.461 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.460+0000 7fc2b1534640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc2ac071730 0x7fc2ac1c5b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:17.461 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.460+0000 7fc2b1534640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fc2ac12f770 0x7fc2ac1c60c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:17.461 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.460+0000 7fc2b1534640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fc2ac134e80 0x7fc2ac1c6600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:17.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.460+0000 7fc2b1534640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7fc2ac138730 con 0x7fc2ac071730 2024-09-21T01:46:17.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.460+0000 7fc2b1534640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7fc2ac1385b0 con 0x7fc2ac12f770 2024-09-21T01:46:17.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.460+0000 7fc2b1534640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7fc2ac1388b0 con 0x7fc2ac134e80 2024-09-21T01:46:17.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.460+0000 7fc2aaffd640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc2ac071730 0x7fc2ac1c5b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:17.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.460+0000 7fc2aa7fc640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fc2ac12f770 0x7fc2ac1c60c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:17.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.460+0000 7fc2aa7fc640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fc2ac12f770 0x7fc2ac1c60c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:60146/0 (socket says 172.21.15.105:60146) 2024-09-21T01:46:17.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.460+0000 7fc2ab7fe640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fc2ac134e80 0x7fc2ac1c6600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:17.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.460+0000 7fc2aa7fc640 1 -- 172.21.15.105:0/2194173965 learned_addr learned my addr 172.21.15.105:0/2194173965 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:17.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.460+0000 7fc2aaffd640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc2ac071730 0x7fc2ac1c5b80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:50762/0 (socket says 172.21.15.105:50762) 2024-09-21T01:46:17.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.460+0000 7fc2aa7fc640 1 -- 172.21.15.105:0/2194173965 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fc2ac134e80 msgr2=0x7fc2ac1c6600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:17.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.460+0000 7fc2aa7fc640 1 --2- 172.21.15.105:0/2194173965 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fc2ac134e80 0x7fc2ac1c6600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.460+0000 7fc2aa7fc640 1 -- 172.21.15.105:0/2194173965 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc2ac071730 msgr2=0x7fc2ac1c5b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:17.462 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.460+0000 7fc2aa7fc640 1 --2- 172.21.15.105:0/2194173965 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc2ac071730 0x7fc2ac1c5b80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.463 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.460+0000 7fc2aa7fc640 1 -- 172.21.15.105:0/2194173965 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fc2ac131520 con 0x7fc2ac12f770 2024-09-21T01:46:17.463 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.460+0000 7fc2aa7fc640 1 --2- 172.21.15.105:0/2194173965 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fc2ac12f770 0x7fc2ac1c60c0 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7fc2a0009980 tx=0x7fc2a0033c20 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:17.463 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.460+0000 7fc2937fe640 1 -- 172.21.15.105:0/2194173965 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc2a0005c90 con 0x7fc2ac12f770 2024-09-21T01:46:17.463 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.460+0000 7fc2937fe640 1 -- 172.21.15.105:0/2194173965 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fc2a0031e50 con 0x7fc2ac12f770 2024-09-21T01:46:17.463 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.460+0000 7fc2937fe640 1 -- 172.21.15.105:0/2194173965 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fc2a0005280 con 0x7fc2ac12f770 2024-09-21T01:46:17.463 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.460+0000 7fc2b1534640 1 -- 172.21.15.105:0/2194173965 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fc2ac130f70 con 0x7fc2ac12f770 2024-09-21T01:46:17.463 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.461+0000 7fc2b1534640 1 -- 172.21.15.105:0/2194173965 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fc2ac09d890 con 0x7fc2ac12f770 2024-09-21T01:46:17.463 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.461+0000 7fc2b1534640 1 -- 172.21.15.105:0/2194173965 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fc2ac09ef00 con 0x7fc2ac12f770 2024-09-21T01:46:17.464 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.462+0000 7fc2937fe640 1 -- 172.21.15.105:0/2194173965 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 9) ==== 63442+0+0 (secure 0 0 0) 0x7fc2a0005420 con 0x7fc2ac12f770 2024-09-21T01:46:17.464 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.462+0000 7fc2937fe640 1 -- 172.21.15.105:0/2194173965 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7fc2a0089ea0 con 0x7fc2ac12f770 2024-09-21T01:46:17.466 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.466+0000 7fc2937fe640 1 -- 172.21.15.105:0/2194173965 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fc2a003f030 con 0x7fc2ac12f770 2024-09-21T01:46:17.642 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.641+0000 7fc2b1534640 1 -- 172.21.15.105:0/2194173965 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "health", "format": "json"} v 0) -- 0x7fc2ac070c50 con 0x7fc2ac12f770 2024-09-21T01:46:17.642 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.641+0000 7fc2937fe640 1 -- 172.21.15.105:0/2194173965 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{"prefix": "health", "format": "json"}]=0 v0) ==== 72+0+46 (secure 0 0 0) 0x7fc2a0031770 con 0x7fc2ac12f770 2024-09-21T01:46:17.642 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:17.642 INFO:teuthology.orchestra.run.smithi105.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T01:46:17.644 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.643+0000 7fc2b1534640 1 -- 172.21.15.105:0/2194173965 >> v1:172.21.15.105:6816/2838005468 conn(0x7fc298051420 legacy=0x7fc298053930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:17.644 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.643+0000 7fc2b1534640 1 -- 172.21.15.105:0/2194173965 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fc2ac12f770 msgr2=0x7fc2ac1c60c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:17.644 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.643+0000 7fc2b1534640 1 --2- 172.21.15.105:0/2194173965 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fc2ac12f770 0x7fc2ac1c60c0 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7fc2a0009980 tx=0x7fc2a0033c20 comp rx=0 tx=0).stop 2024-09-21T01:46:17.644 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.643+0000 7fc2b1534640 1 -- 172.21.15.105:0/2194173965 shutdown_connections 2024-09-21T01:46:17.644 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.643+0000 7fc2b1534640 1 --2- 172.21.15.105:0/2194173965 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fc2ac134e80 0x7fc2ac1c6600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.644 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.643+0000 7fc2b1534640 1 --2- 172.21.15.105:0/2194173965 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fc2ac12f770 0x7fc2ac1c60c0 unknown :-1 s=CLOSED pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.644 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.643+0000 7fc2b1534640 1 --2- 172.21.15.105:0/2194173965 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fc2ac071730 0x7fc2ac1c5b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.644 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.643+0000 7fc2b1534640 1 -- 172.21.15.105:0/2194173965 >> 172.21.15.105:0/2194173965 conn(0x7fc2ac0a2080 msgr2=0x7fc2ac073580 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:17.644 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.643+0000 7fc2b1534640 1 -- 172.21.15.105:0/2194173965 shutdown_connections 2024-09-21T01:46:17.644 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.643+0000 7fc2b1534640 1 -- 172.21.15.105:0/2194173965 wait complete. 2024-09-21T01:46:17.654 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy done 2024-09-21T01:46:17.654 INFO:teuthology.run_tasks:Running task ceph.restart... 2024-09-21T01:46:17.666 DEBUG:tasks.ceph.osd.0:waiting for process to exit 2024-09-21T01:46:17.666 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T01:46:17.702 INFO:tasks.ceph.osd.0:Stopped 2024-09-21T01:46:17.703 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd down 0 2024-09-21T01:46:17.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.852+0000 7f4b20b38640 1 -- 172.21.15.105:0/3655770102 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4b1c0727d0 msgr2=0x7f4b1c12c800 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:17.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.852+0000 7f4b20b38640 1 --2- 172.21.15.105:0/3655770102 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4b1c0727d0 0x7f4b1c12c800 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f4b14005fd0 tx=0x7f4b1402a780 comp rx=0 tx=0).stop 2024-09-21T01:46:17.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.852+0000 7f4b20b38640 1 -- 172.21.15.105:0/3655770102 shutdown_connections 2024-09-21T01:46:17.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.852+0000 7f4b20b38640 1 --2- 172.21.15.105:0/3655770102 >> v2:172.21.15.105:3300/0 conn(0x7f4b1c12c2d0 0x7f4b1c12cd40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.852+0000 7f4b20b38640 1 --2- 172.21.15.105:0/3655770102 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4b1c0727d0 0x7f4b1c12c800 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.852+0000 7f4b20b38640 1 -- 172.21.15.105:0/3655770102 >> 172.21.15.105:0/3655770102 conn(0x7f4b1c09eb40 msgr2=0x7f4b1c09ef50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:17.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.852+0000 7f4b20b38640 1 -- 172.21.15.105:0/3655770102 shutdown_connections 2024-09-21T01:46:17.853 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.852+0000 7f4b20b38640 1 -- 172.21.15.105:0/3655770102 wait complete. 2024-09-21T01:46:17.854 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.853+0000 7f4b20b38640 1 Processor -- start 2024-09-21T01:46:17.854 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.854+0000 7f4b20b38640 1 -- start start 2024-09-21T01:46:17.856 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.854+0000 7f4b20b38640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4b1c0727d0 0x7f4b1c133a20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:17.856 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.854+0000 7f4b20b38640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4b1c128dd0 0x7f4b1c133f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:17.856 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.854+0000 7f4b20b38640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4b1c12c2d0 0x7f4b1c06e9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:17.856 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.854+0000 7f4b20b38640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f4b1c1352d0 con 0x7f4b1c128dd0 2024-09-21T01:46:17.856 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.854+0000 7f4b20b38640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f4b1c135150 con 0x7f4b1c12c2d0 2024-09-21T01:46:17.856 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.854+0000 7f4b20b38640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f4b1c135450 con 0x7f4b1c0727d0 2024-09-21T01:46:17.856 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.854+0000 7f4b1a59c640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4b1c0727d0 0x7f4b1c133a20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:17.856 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.854+0000 7f4b19d9b640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4b1c128dd0 0x7f4b1c133f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:17.856 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.854+0000 7f4b1a59c640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4b1c0727d0 0x7f4b1c133a20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:50366/0 (socket says 172.21.15.105:50366) 2024-09-21T01:46:17.856 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.854+0000 7f4b1ad9d640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4b1c12c2d0 0x7f4b1c06e9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:17.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.854+0000 7f4b1a59c640 1 -- 172.21.15.105:0/3801600538 learned_addr learned my addr 172.21.15.105:0/3801600538 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:17.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.855+0000 7f4b19d9b640 1 -- 172.21.15.105:0/3801600538 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4b1c0727d0 msgr2=0x7f4b1c133a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:17.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.855+0000 7f4b19d9b640 1 --2- 172.21.15.105:0/3801600538 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4b1c0727d0 0x7f4b1c133a20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.855+0000 7f4b19d9b640 1 -- 172.21.15.105:0/3801600538 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4b1c12c2d0 msgr2=0x7f4b1c06e9e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:17.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.855+0000 7f4b19d9b640 1 --2- 172.21.15.105:0/3801600538 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4b1c12c2d0 0x7f4b1c06e9e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:17.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.855+0000 7f4b19d9b640 1 -- 172.21.15.105:0/3801600538 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4b1c095670 con 0x7f4b1c128dd0 2024-09-21T01:46:17.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.855+0000 7f4b1a59c640 1 --2- 172.21.15.105:0/3801600538 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4b1c0727d0 0x7f4b1c133a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:17.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.855+0000 7f4b19d9b640 1 --2- 172.21.15.105:0/3801600538 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4b1c128dd0 0x7f4b1c133f60 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f4b0400ea20 tx=0x7f4b0400eee0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:17.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.855+0000 7f4b12ffd640 1 -- 172.21.15.105:0/3801600538 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4b0400cc60 con 0x7f4b1c128dd0 2024-09-21T01:46:17.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.855+0000 7f4b12ffd640 1 -- 172.21.15.105:0/3801600538 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f4b0400ce00 con 0x7f4b1c128dd0 2024-09-21T01:46:17.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.855+0000 7f4b20b38640 1 -- 172.21.15.105:0/3801600538 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4b1c095f10 con 0x7f4b1c128dd0 2024-09-21T01:46:17.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.855+0000 7f4b12ffd640 1 -- 172.21.15.105:0/3801600538 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4b04004280 con 0x7f4b1c128dd0 2024-09-21T01:46:17.857 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.855+0000 7f4b20b38640 1 -- 172.21.15.105:0/3801600538 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f4b1c06ef20 con 0x7f4b1c128dd0 2024-09-21T01:46:17.858 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.856+0000 7f4b20b38640 1 -- 172.21.15.105:0/3801600538 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4b1c1291e0 con 0x7f4b1c128dd0 2024-09-21T01:46:17.858 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.856+0000 7f4b12ffd640 1 -- 172.21.15.105:0/3801600538 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 9) ==== 63442+0+0 (secure 0 0 0) 0x7f4b040035f0 con 0x7f4b1c128dd0 2024-09-21T01:46:17.858 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.857+0000 7f4b12ffd640 1 -- 172.21.15.105:0/3801600538 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(20..20 src has 1..20) ==== 3027+0+0 (secure 0 0 0) 0x7f4b04069820 con 0x7f4b1c128dd0 2024-09-21T01:46:17.860 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.860+0000 7f4b12ffd640 1 -- 172.21.15.105:0/3801600538 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f4b04004420 con 0x7f4b1c128dd0 2024-09-21T01:46:17.992 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:17.992+0000 7f4b12ffd640 1 -- 172.21.15.105:0/3801600538 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f4b04068480 con 0x7f4b1c128dd0 2024-09-21T01:46:18.024 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.023+0000 7f4b20b38640 1 -- 172.21.15.105:0/3801600538 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd down", "ids": ["0"]} v 0) -- 0x7f4b1c06a780 con 0x7f4b1c128dd0 2024-09-21T01:46:18.024 INFO:tasks.ceph.mon.a.smithi105.stderr:2024-09-21T01:46:18.024+0000 7f147cdfa640 -1 mon.a@0(leader).osd e21 definitely_dead 0 2024-09-21T01:46:18.025 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.024+0000 7f4b12ffd640 1 -- 172.21.15.105:0/3801600538 <== mon.0 v2:172.21.15.105:3300/0 8 ==== mon_command_ack([{"prefix": "osd down", "ids": ["0"]}]=0 osd.0 is already down. v21) ==== 93+0+0 (secure 0 0 0) 0x7f4b04004220 con 0x7f4b1c128dd0 2024-09-21T01:46:18.025 INFO:teuthology.orchestra.run.smithi105.stderr:osd.0 is already down. 2024-09-21T01:46:18.026 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.025+0000 7f4b20b38640 1 -- 172.21.15.105:0/3801600538 >> v1:172.21.15.105:6816/2838005468 conn(0x7f4afc051420 legacy=0x7f4afc053930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:18.026 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.026+0000 7f4b20b38640 1 -- 172.21.15.105:0/3801600538 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4b1c128dd0 msgr2=0x7f4b1c133f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:18.026 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.026+0000 7f4b20b38640 1 --2- 172.21.15.105:0/3801600538 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4b1c128dd0 0x7f4b1c133f60 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f4b0400ea20 tx=0x7f4b0400eee0 comp rx=0 tx=0).stop 2024-09-21T01:46:18.026 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.026+0000 7f4b20b38640 1 -- 172.21.15.105:0/3801600538 shutdown_connections 2024-09-21T01:46:18.027 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.026+0000 7f4b20b38640 1 --2- 172.21.15.105:0/3801600538 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4b1c12c2d0 0x7f4b1c06e9e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:18.027 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.026+0000 7f4b20b38640 1 --2- 172.21.15.105:0/3801600538 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4b1c128dd0 0x7f4b1c133f60 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:18.027 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.026+0000 7f4b20b38640 1 --2- 172.21.15.105:0/3801600538 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4b1c0727d0 0x7f4b1c133a20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:18.027 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.026+0000 7f4b20b38640 1 -- 172.21.15.105:0/3801600538 >> 172.21.15.105:0/3801600538 conn(0x7f4b1c09eb40 msgr2=0x7f4b1c0985f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:18.027 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.026+0000 7f4b20b38640 1 -- 172.21.15.105:0/3801600538 shutdown_connections 2024-09-21T01:46:18.027 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.026+0000 7f4b20b38640 1 -- 172.21.15.105:0/3801600538 wait complete. 2024-09-21T01:46:18.037 INFO:tasks.ceph.osd.0:Restarting daemon 2024-09-21T01:46:18.037 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 0 2024-09-21T01:46:18.039 INFO:tasks.ceph.osd.0:Started 2024-09-21T01:46:18.040 DEBUG:tasks.ceph.osd.1:waiting for process to exit 2024-09-21T01:46:18.040 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T01:46:18.070 INFO:tasks.ceph.osd.1:Stopped 2024-09-21T01:46:18.070 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd down 1 2024-09-21T01:46:18.223 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.222+0000 7f3c736af640 1 -- 172.21.15.105:0/165831052 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3c6c0960e0 msgr2=0x7f3c6c1377d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:18.223 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.222+0000 7f3c736af640 1 --2- 172.21.15.105:0/165831052 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3c6c0960e0 0x7f3c6c1377d0 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7f3c64009be0 tx=0x7f3c6402a800 comp rx=0 tx=0).stop 2024-09-21T01:46:18.223 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.222+0000 7f3c736af640 1 -- 172.21.15.105:0/165831052 shutdown_connections 2024-09-21T01:46:18.223 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.222+0000 7f3c736af640 1 --2- 172.21.15.105:0/165831052 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3c6c0960e0 0x7f3c6c1377d0 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:18.223 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.222+0000 7f3c736af640 1 --2- 172.21.15.105:0/165831052 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3c6c095cd0 0x7f3c6c135280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:18.223 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.222+0000 7f3c736af640 1 -- 172.21.15.105:0/165831052 >> 172.21.15.105:0/165831052 conn(0x7f3c6c09e2a0 msgr2=0x7f3c6c09e6b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:18.223 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.222+0000 7f3c736af640 1 -- 172.21.15.105:0/165831052 shutdown_connections 2024-09-21T01:46:18.223 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.222+0000 7f3c736af640 1 -- 172.21.15.105:0/165831052 wait complete. 2024-09-21T01:46:18.225 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.223+0000 7f3c736af640 1 Processor -- start 2024-09-21T01:46:18.226 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.223+0000 7f3c736af640 1 -- start start 2024-09-21T01:46:18.226 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.224+0000 7f3c736af640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3c6c094d50 0x7f3c6c125710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:18.226 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.224+0000 7f3c736af640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3c6c095cd0 0x7f3c6c125c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:18.226 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.224+0000 7f3c736af640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3c6c0960e0 0x7f3c6c126190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:18.226 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.224+0000 7f3c736af640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f3c6c138870 con 0x7f3c6c0960e0 2024-09-21T01:46:18.226 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.224+0000 7f3c736af640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f3c6c1386f0 con 0x7f3c6c094d50 2024-09-21T01:46:18.226 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.224+0000 7f3c736af640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f3c6c1389f0 con 0x7f3c6c095cd0 2024-09-21T01:46:18.226 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.224+0000 7f3c70c4a640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3c6c095cd0 0x7f3c6c125c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:18.226 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.224+0000 7f3c7144b640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3c6c094d50 0x7f3c6c125710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:18.226 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.224+0000 7f3c70c4a640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3c6c095cd0 0x7f3c6c125c50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:50390/0 (socket says 172.21.15.105:50390) 2024-09-21T01:46:18.227 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.224+0000 7f3c71c4c640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3c6c0960e0 0x7f3c6c126190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:18.227 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.224+0000 7f3c70c4a640 1 -- 172.21.15.105:0/1451598896 learned_addr learned my addr 172.21.15.105:0/1451598896 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:18.227 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.224+0000 7f3c7144b640 1 -- 172.21.15.105:0/1451598896 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3c6c095cd0 msgr2=0x7f3c6c125c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:18.227 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.224+0000 7f3c7144b640 1 --2- 172.21.15.105:0/1451598896 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3c6c095cd0 0x7f3c6c125c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:18.227 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.224+0000 7f3c7144b640 1 -- 172.21.15.105:0/1451598896 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3c6c0960e0 msgr2=0x7f3c6c126190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:18.227 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.224+0000 7f3c7144b640 1 --2- 172.21.15.105:0/1451598896 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3c6c0960e0 0x7f3c6c126190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:18.227 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.224+0000 7f3c7144b640 1 -- 172.21.15.105:0/1451598896 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f3c6c1278b0 con 0x7f3c6c094d50 2024-09-21T01:46:18.227 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.224+0000 7f3c70c4a640 1 --2- 172.21.15.105:0/1451598896 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3c6c095cd0 0x7f3c6c125c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:18.227 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.224+0000 7f3c71c4c640 1 --2- 172.21.15.105:0/1451598896 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3c6c0960e0 0x7f3c6c126190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:18.227 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.224+0000 7f3c7144b640 1 --2- 172.21.15.105:0/1451598896 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3c6c094d50 0x7f3c6c125710 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7f3c6800c390 tx=0x7f3c6800c850 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:18.227 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.224+0000 7f3c61ffb640 1 -- 172.21.15.105:0/1451598896 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3c68011790 con 0x7f3c6c094d50 2024-09-21T01:46:18.227 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.224+0000 7f3c61ffb640 1 -- 172.21.15.105:0/1451598896 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f3c68017070 con 0x7f3c6c094d50 2024-09-21T01:46:18.227 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.224+0000 7f3c736af640 1 -- 172.21.15.105:0/1451598896 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f3c6c128700 con 0x7f3c6c094d50 2024-09-21T01:46:18.227 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.225+0000 7f3c61ffb640 1 -- 172.21.15.105:0/1451598896 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3c68011ce0 con 0x7f3c6c094d50 2024-09-21T01:46:18.228 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.225+0000 7f3c736af640 1 -- 172.21.15.105:0/1451598896 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f3c6c128150 con 0x7f3c6c094d50 2024-09-21T01:46:18.228 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.226+0000 7f3c736af640 1 -- 172.21.15.105:0/1451598896 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f3c6c0955c0 con 0x7f3c6c094d50 2024-09-21T01:46:18.228 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.226+0000 7f3c61ffb640 1 -- 172.21.15.105:0/1451598896 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f3c6801c460 con 0x7f3c6c094d50 2024-09-21T01:46:18.228 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.226+0000 7f3c61ffb640 1 -- 172.21.15.105:0/1451598896 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(21..21 src has 1..21) ==== 3027+0+0 (secure 0 0 0) 0x7f3c68068cb0 con 0x7f3c6c094d50 2024-09-21T01:46:18.230 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.230+0000 7f3c61ffb640 1 -- 172.21.15.105:0/1451598896 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f3c68067d60 con 0x7f3c6c094d50 2024-09-21T01:46:18.390 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:18.390+0000 7f3c736af640 1 -- 172.21.15.105:0/1451598896 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "osd down", "ids": ["1"]} v 0) -- 0x7f3c6c098590 con 0x7f3c6c094d50 2024-09-21T01:46:18.391 INFO:tasks.ceph.mon.a.smithi105.stderr:2024-09-21T01:46:18.391+0000 7f147cdfa640 -1 mon.a@0(leader).osd e21 definitely_dead 0 2024-09-21T01:46:19.001 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.000+0000 7f3c61ffb640 1 -- 172.21.15.105:0/1451598896 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{"prefix": "osd down", "ids": ["1"]}]=0 marked down osd.1. v22) ==== 89+0+0 (secure 0 0 0) 0x7f3c68067520 con 0x7f3c6c094d50 2024-09-21T01:46:19.001 INFO:teuthology.orchestra.run.smithi105.stderr:marked down osd.1. 2024-09-21T01:46:19.003 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.002+0000 7f3c736af640 1 -- 172.21.15.105:0/1451598896 >> v1:172.21.15.105:6816/2838005468 conn(0x7f3c500514b0 legacy=0x7f3c500539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:19.003 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.003+0000 7f3c736af640 1 -- 172.21.15.105:0/1451598896 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3c6c094d50 msgr2=0x7f3c6c125710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:19.003 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.003+0000 7f3c736af640 1 --2- 172.21.15.105:0/1451598896 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3c6c094d50 0x7f3c6c125710 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7f3c6800c390 tx=0x7f3c6800c850 comp rx=0 tx=0).stop 2024-09-21T01:46:19.003 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.003+0000 7f3c736af640 1 -- 172.21.15.105:0/1451598896 shutdown_connections 2024-09-21T01:46:19.004 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.003+0000 7f3c736af640 1 --2- 172.21.15.105:0/1451598896 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f3c6c0960e0 0x7f3c6c126190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:19.004 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.003+0000 7f3c736af640 1 --2- 172.21.15.105:0/1451598896 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f3c6c095cd0 0x7f3c6c125c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:19.004 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.003+0000 7f3c736af640 1 --2- 172.21.15.105:0/1451598896 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f3c6c094d50 0x7f3c6c125710 unknown :-1 s=CLOSED pgs=34 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:19.004 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.003+0000 7f3c736af640 1 -- 172.21.15.105:0/1451598896 >> 172.21.15.105:0/1451598896 conn(0x7f3c6c09e2a0 msgr2=0x7f3c6c093e50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:19.004 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.003+0000 7f3c736af640 1 -- 172.21.15.105:0/1451598896 shutdown_connections 2024-09-21T01:46:19.004 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.003+0000 7f3c736af640 1 -- 172.21.15.105:0/1451598896 wait complete. 2024-09-21T01:46:19.016 INFO:tasks.ceph.osd.1:Restarting daemon 2024-09-21T01:46:19.016 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 1 2024-09-21T01:46:19.018 INFO:tasks.ceph.osd.1:Started 2024-09-21T01:46:19.019 DEBUG:tasks.ceph.osd.2:waiting for process to exit 2024-09-21T01:46:19.019 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T01:46:19.057 INFO:tasks.ceph.osd.2:Stopped 2024-09-21T01:46:19.057 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd down 2 2024-09-21T01:46:19.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.208+0000 7f6b468f6640 1 -- 172.21.15.105:0/2660609192 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6b40073b30 msgr2=0x7f6b4012d910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:19.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.208+0000 7f6b468f6640 1 --2- 172.21.15.105:0/2660609192 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6b40073b30 0x7f6b4012d910 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7f6b30009860 tx=0x7f6b3002a460 comp rx=0 tx=0).stop 2024-09-21T01:46:19.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.208+0000 7f6b468f6640 1 -- 172.21.15.105:0/2660609192 shutdown_connections 2024-09-21T01:46:19.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.208+0000 7f6b468f6640 1 --2- 172.21.15.105:0/2660609192 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6b4012a5d0 0x7f6b401357c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:19.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.208+0000 7f6b468f6640 1 --2- 172.21.15.105:0/2660609192 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6b40073b30 0x7f6b4012d910 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:19.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.208+0000 7f6b468f6640 1 -- 172.21.15.105:0/2660609192 >> 172.21.15.105:0/2660609192 conn(0x7f6b4009fe70 msgr2=0x7f6b400a0280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:19.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.208+0000 7f6b468f6640 1 -- 172.21.15.105:0/2660609192 shutdown_connections 2024-09-21T01:46:19.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.208+0000 7f6b468f6640 1 -- 172.21.15.105:0/2660609192 wait complete. 2024-09-21T01:46:19.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.209+0000 7f6b468f6640 1 Processor -- start 2024-09-21T01:46:19.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.210+0000 7f6b468f6640 1 -- start start 2024-09-21T01:46:19.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.210+0000 7f6b468f6640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6b40073b30 0x7f6b400965c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:19.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.210+0000 7f6b468f6640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6b4012a5d0 0x7f6b40096b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:19.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.210+0000 7f6b468f6640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6b4012de50 0x7f6b40097040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:19.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.210+0000 7f6b468f6640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f6b40136500 con 0x7f6b4012de50 2024-09-21T01:46:19.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.210+0000 7f6b468f6640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f6b40136380 con 0x7f6b4012a5d0 2024-09-21T01:46:19.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.210+0000 7f6b468f6640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f6b40136680 con 0x7f6b40073b30 2024-09-21T01:46:19.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.210+0000 7f6b3ffff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6b40073b30 0x7f6b400965c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:19.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.210+0000 7f6b3ffff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6b40073b30 0x7f6b400965c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:50416/0 (socket says 172.21.15.105:50416) 2024-09-21T01:46:19.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.210+0000 7f6b44e93640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6b4012de50 0x7f6b40097040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:19.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.210+0000 7f6b3ffff640 1 -- 172.21.15.105:0/1002852887 learned_addr learned my addr 172.21.15.105:0/1002852887 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:19.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.210+0000 7f6b3f7fe640 1 --2- 172.21.15.105:0/1002852887 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6b4012a5d0 0x7f6b40096b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:19.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.211+0000 7f6b3ffff640 1 -- 172.21.15.105:0/1002852887 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6b4012a5d0 msgr2=0x7f6b40096b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:19.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.211+0000 7f6b3ffff640 1 --2- 172.21.15.105:0/1002852887 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6b4012a5d0 0x7f6b40096b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:19.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.211+0000 7f6b3ffff640 1 -- 172.21.15.105:0/1002852887 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6b4012de50 msgr2=0x7f6b40097040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:19.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.211+0000 7f6b3ffff640 1 --2- 172.21.15.105:0/1002852887 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6b4012de50 0x7f6b40097040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:19.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.211+0000 7f6b3ffff640 1 -- 172.21.15.105:0/1002852887 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6b40098000 con 0x7f6b40073b30 2024-09-21T01:46:19.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.211+0000 7f6b44e93640 1 --2- 172.21.15.105:0/1002852887 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6b4012de50 0x7f6b40097040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:19.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.211+0000 7f6b3ffff640 1 --2- 172.21.15.105:0/1002852887 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6b40073b30 0x7f6b400965c0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f6b30009780 tx=0x7f6b30033ba0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:19.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.211+0000 7f6b3cff9640 1 -- 172.21.15.105:0/1002852887 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b30031b50 con 0x7f6b40073b30 2024-09-21T01:46:19.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.211+0000 7f6b3cff9640 1 -- 172.21.15.105:0/1002852887 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f6b30031e00 con 0x7f6b40073b30 2024-09-21T01:46:19.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.211+0000 7f6b468f6640 1 -- 172.21.15.105:0/1002852887 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6b40098e00 con 0x7f6b40073b30 2024-09-21T01:46:19.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.211+0000 7f6b3cff9640 1 -- 172.21.15.105:0/1002852887 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6b30005280 con 0x7f6b40073b30 2024-09-21T01:46:19.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.211+0000 7f6b468f6640 1 -- 172.21.15.105:0/1002852887 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f6b401c9980 con 0x7f6b40073b30 2024-09-21T01:46:19.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.212+0000 7f6b468f6640 1 -- 172.21.15.105:0/1002852887 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6b4012d910 con 0x7f6b40073b30 2024-09-21T01:46:19.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.212+0000 7f6b3cff9640 1 -- 172.21.15.105:0/1002852887 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f6b30005420 con 0x7f6b40073b30 2024-09-21T01:46:19.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.213+0000 7f6b3cff9640 1 -- 172.21.15.105:0/1002852887 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(22..22 src has 1..22) ==== 3027+0+0 (secure 0 0 0) 0x7f6b30089fa0 con 0x7f6b40073b30 2024-09-21T01:46:19.217 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.216+0000 7f6b3cff9640 1 -- 172.21.15.105:0/1002852887 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f6b3003f030 con 0x7f6b40073b30 2024-09-21T01:46:19.378 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:19.377+0000 7f6b468f6640 1 -- 172.21.15.105:0/1002852887 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd down", "ids": ["2"]} v 0) -- 0x7f6b4006a740 con 0x7f6b40073b30 2024-09-21T01:46:19.378 INFO:tasks.ceph.mon.a.smithi105.stderr:2024-09-21T01:46:19.378+0000 7f147cdfa640 -1 mon.a@0(leader).osd e22 definitely_dead 0 2024-09-21T01:46:19.721 INFO:tasks.ceph.osd.0.smithi105.stderr:2024-09-21T01:46:19.720+0000 7f236d20b640 -1 Falling back to public interface 2024-09-21T01:46:20.009 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.009+0000 7f6b3cff9640 1 -- 172.21.15.105:0/1002852887 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd down", "ids": ["2"]}]=0 marked down osd.2. v23) ==== 89+0+0 (secure 0 0 0) 0x7f6b30089050 con 0x7f6b40073b30 2024-09-21T01:46:20.009 INFO:teuthology.orchestra.run.smithi105.stderr:marked down osd.2. 2024-09-21T01:46:20.012 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.011+0000 7f6b468f6640 1 -- 172.21.15.105:0/1002852887 >> v1:172.21.15.105:6816/2838005468 conn(0x7f6b2c0514b0 legacy=0x7f6b2c0539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:20.012 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.011+0000 7f6b468f6640 1 -- 172.21.15.105:0/1002852887 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6b40073b30 msgr2=0x7f6b400965c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:20.012 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.011+0000 7f6b468f6640 1 --2- 172.21.15.105:0/1002852887 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6b40073b30 0x7f6b400965c0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7f6b30009780 tx=0x7f6b30033ba0 comp rx=0 tx=0).stop 2024-09-21T01:46:20.012 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.011+0000 7f6b468f6640 1 -- 172.21.15.105:0/1002852887 shutdown_connections 2024-09-21T01:46:20.012 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.011+0000 7f6b468f6640 1 --2- 172.21.15.105:0/1002852887 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6b4012de50 0x7f6b40097040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:20.012 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.011+0000 7f6b468f6640 1 --2- 172.21.15.105:0/1002852887 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6b4012a5d0 0x7f6b40096b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:20.012 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.011+0000 7f6b468f6640 1 --2- 172.21.15.105:0/1002852887 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6b40073b30 0x7f6b400965c0 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:20.012 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.011+0000 7f6b468f6640 1 -- 172.21.15.105:0/1002852887 >> 172.21.15.105:0/1002852887 conn(0x7f6b4009fe70 msgr2=0x7f6b4012c090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:20.012 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.011+0000 7f6b468f6640 1 -- 172.21.15.105:0/1002852887 shutdown_connections 2024-09-21T01:46:20.012 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.011+0000 7f6b468f6640 1 -- 172.21.15.105:0/1002852887 wait complete. 2024-09-21T01:46:20.023 INFO:tasks.ceph.osd.2:Restarting daemon 2024-09-21T01:46:20.023 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 2 2024-09-21T01:46:20.025 INFO:tasks.ceph.osd.2:Started 2024-09-21T01:46:20.026 DEBUG:tasks.ceph.osd.3:waiting for process to exit 2024-09-21T01:46:20.026 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T01:46:20.059 INFO:tasks.ceph.osd.3:Stopped 2024-09-21T01:46:20.059 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd down 3 2024-09-21T01:46:20.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.210+0000 7ff6b19ae640 1 -- 172.21.15.105:0/1129919558 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff6ac071730 msgr2=0x7ff6ac134940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:20.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.210+0000 7ff6b19ae640 1 --2- 172.21.15.105:0/1129919558 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff6ac071730 0x7ff6ac134940 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7ff6a0009860 tx=0x7ff6a002a460 comp rx=0 tx=0).stop 2024-09-21T01:46:20.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.210+0000 7ff6b19ae640 1 -- 172.21.15.105:0/1129919558 shutdown_connections 2024-09-21T01:46:20.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.210+0000 7ff6b19ae640 1 --2- 172.21.15.105:0/1129919558 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff6ac134e80 0x7ff6ac1372a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:20.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.210+0000 7ff6b19ae640 1 --2- 172.21.15.105:0/1129919558 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff6ac071730 0x7ff6ac134940 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:20.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.210+0000 7ff6b19ae640 1 -- 172.21.15.105:0/1129919558 >> 172.21.15.105:0/1129919558 conn(0x7ff6ac0a2080 msgr2=0x7ff6ac0a2490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:20.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.210+0000 7ff6b19ae640 1 -- 172.21.15.105:0/1129919558 shutdown_connections 2024-09-21T01:46:20.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.210+0000 7ff6b19ae640 1 -- 172.21.15.105:0/1129919558 wait complete. 2024-09-21T01:46:20.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.211+0000 7ff6b19ae640 1 Processor -- start 2024-09-21T01:46:20.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.211+0000 7ff6b19ae640 1 -- start start 2024-09-21T01:46:20.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.211+0000 7ff6b19ae640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff6ac071730 0x7ff6ac1c5b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:20.213 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.211+0000 7ff6b19ae640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff6ac12f770 0x7ff6ac1c60c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:20.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.211+0000 7ff6b19ae640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff6ac134e80 0x7ff6ac1c6600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:20.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.211+0000 7ff6b19ae640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7ff6ac138730 con 0x7ff6ac12f770 2024-09-21T01:46:20.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.212+0000 7ff6b19ae640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7ff6ac1385b0 con 0x7ff6ac134e80 2024-09-21T01:46:20.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.212+0000 7ff6b19ae640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7ff6ac1388b0 con 0x7ff6ac071730 2024-09-21T01:46:20.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.212+0000 7ff6aa7fc640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff6ac12f770 0x7ff6ac1c60c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:20.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.212+0000 7ff6aa7fc640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff6ac12f770 0x7ff6ac1c60c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:50828/0 (socket says 172.21.15.105:50828) 2024-09-21T01:46:20.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.212+0000 7ff6ab7fe640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff6ac134e80 0x7ff6ac1c6600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:20.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.212+0000 7ff6aa7fc640 1 -- 172.21.15.105:0/356157779 learned_addr learned my addr 172.21.15.105:0/356157779 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:20.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.212+0000 7ff6aaffd640 1 --2- 172.21.15.105:0/356157779 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff6ac071730 0x7ff6ac1c5b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:20.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.212+0000 7ff6aa7fc640 1 -- 172.21.15.105:0/356157779 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff6ac071730 msgr2=0x7ff6ac1c5b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:20.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.212+0000 7ff6aa7fc640 1 --2- 172.21.15.105:0/356157779 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff6ac071730 0x7ff6ac1c5b80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:20.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.212+0000 7ff6aa7fc640 1 -- 172.21.15.105:0/356157779 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff6ac134e80 msgr2=0x7ff6ac1c6600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:20.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.212+0000 7ff6aa7fc640 1 --2- 172.21.15.105:0/356157779 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff6ac134e80 0x7ff6ac1c6600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:20.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.212+0000 7ff6aa7fc640 1 -- 172.21.15.105:0/356157779 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff6ac1306d0 con 0x7ff6ac12f770 2024-09-21T01:46:20.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.212+0000 7ff6ab7fe640 1 --2- 172.21.15.105:0/356157779 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff6ac134e80 0x7ff6ac1c6600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:20.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.212+0000 7ff6aa7fc640 1 --2- 172.21.15.105:0/356157779 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff6ac12f770 0x7ff6ac1c60c0 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7ff6a0009980 tx=0x7ff6a0033ba0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:20.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.212+0000 7ff693fff640 1 -- 172.21.15.105:0/356157779 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6a0031b50 con 0x7ff6ac12f770 2024-09-21T01:46:20.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.212+0000 7ff693fff640 1 -- 172.21.15.105:0/356157779 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7ff6a0031e00 con 0x7ff6ac12f770 2024-09-21T01:46:20.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.212+0000 7ff693fff640 1 -- 172.21.15.105:0/356157779 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff6a0005280 con 0x7ff6ac12f770 2024-09-21T01:46:20.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.212+0000 7ff6b19ae640 1 -- 172.21.15.105:0/356157779 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff6ac130c10 con 0x7ff6ac12f770 2024-09-21T01:46:20.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.213+0000 7ff6b19ae640 1 -- 172.21.15.105:0/356157779 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff6ac12aee0 con 0x7ff6ac12f770 2024-09-21T01:46:20.215 INFO:tasks.ceph.osd.1.smithi105.stderr:2024-09-21T01:46:20.213+0000 7fb080ba8640 -1 Falling back to public interface 2024-09-21T01:46:20.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.214+0000 7ff693fff640 1 -- 172.21.15.105:0/356157779 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7ff6a0005420 con 0x7ff6ac12f770 2024-09-21T01:46:20.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.214+0000 7ff6b19ae640 1 -- 172.21.15.105:0/356157779 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff6ac070c50 con 0x7ff6ac12f770 2024-09-21T01:46:20.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.214+0000 7ff693fff640 1 -- 172.21.15.105:0/356157779 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(23..23 src has 1..23) ==== 3027+0+0 (secure 0 0 0) 0x7ff6a008a040 con 0x7ff6ac12f770 2024-09-21T01:46:20.218 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.218+0000 7ff693fff640 1 -- 172.21.15.105:0/356157779 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7ff6a003f030 con 0x7ff6ac12f770 2024-09-21T01:46:20.378 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:20.378+0000 7ff6b19ae640 1 -- 172.21.15.105:0/356157779 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd down", "ids": ["3"]} v 0) -- 0x7ff6ac12c640 con 0x7ff6ac12f770 2024-09-21T01:46:20.379 INFO:tasks.ceph.mon.a.smithi105.stderr:2024-09-21T01:46:20.378+0000 7f147cdfa640 -1 mon.a@0(leader).osd e23 definitely_dead 0 2024-09-21T01:46:21.002 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.002+0000 7ff693fff640 1 -- 172.21.15.105:0/356157779 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd down", "ids": ["3"]}]=0 marked down osd.3. v24) ==== 89+0+0 (secure 0 0 0) 0x7ff6a00890f0 con 0x7ff6ac12f770 2024-09-21T01:46:21.002 INFO:teuthology.orchestra.run.smithi105.stderr:marked down osd.3. 2024-09-21T01:46:21.004 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.004+0000 7ff6b19ae640 1 -- 172.21.15.105:0/356157779 >> v1:172.21.15.105:6816/2838005468 conn(0x7ff698051510 legacy=0x7ff698053a20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:21.005 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.004+0000 7ff6b19ae640 1 -- 172.21.15.105:0/356157779 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff6ac12f770 msgr2=0x7ff6ac1c60c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:21.005 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.004+0000 7ff6b19ae640 1 --2- 172.21.15.105:0/356157779 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff6ac12f770 0x7ff6ac1c60c0 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7ff6a0009980 tx=0x7ff6a0033ba0 comp rx=0 tx=0).stop 2024-09-21T01:46:21.005 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.004+0000 7ff6b19ae640 1 -- 172.21.15.105:0/356157779 shutdown_connections 2024-09-21T01:46:21.005 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.004+0000 7ff6b19ae640 1 --2- 172.21.15.105:0/356157779 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff6ac134e80 0x7ff6ac1c6600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:21.005 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.004+0000 7ff6b19ae640 1 --2- 172.21.15.105:0/356157779 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff6ac12f770 0x7ff6ac1c60c0 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:21.005 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.004+0000 7ff6b19ae640 1 --2- 172.21.15.105:0/356157779 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff6ac071730 0x7ff6ac1c5b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:21.005 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.004+0000 7ff6b19ae640 1 -- 172.21.15.105:0/356157779 >> 172.21.15.105:0/356157779 conn(0x7ff6ac0a2080 msgr2=0x7ff6ac073610 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:21.005 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.004+0000 7ff6b19ae640 1 -- 172.21.15.105:0/356157779 shutdown_connections 2024-09-21T01:46:21.005 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.004+0000 7ff6b19ae640 1 -- 172.21.15.105:0/356157779 wait complete. 2024-09-21T01:46:21.018 INFO:tasks.ceph.osd.3:Restarting daemon 2024-09-21T01:46:21.018 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster ceph -i 3 2024-09-21T01:46:21.020 INFO:tasks.ceph.osd.3:Started 2024-09-21T01:46:21.021 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up 2024-09-21T01:46:21.021 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T01:46:21.134 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.134+0000 7fa831907640 1 -- 172.21.15.105:0/2210461232 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa82c070c50 msgr2=0x7fa82c12c060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:21.135 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.134+0000 7fa831907640 1 --2- 172.21.15.105:0/2210461232 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa82c070c50 0x7fa82c12c060 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7fa814009830 tx=0x7fa81402a480 comp rx=0 tx=0).stop 2024-09-21T01:46:21.135 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.134+0000 7fa831907640 1 -- 172.21.15.105:0/2210461232 shutdown_connections 2024-09-21T01:46:21.135 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.134+0000 7fa831907640 1 --2- 172.21.15.105:0/2210461232 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa82c070c50 0x7fa82c12c060 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:21.135 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.134+0000 7fa831907640 1 --2- 172.21.15.105:0/2210461232 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa82c12f770 0x7fa82c071730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:21.135 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.134+0000 7fa831907640 1 -- 172.21.15.105:0/2210461232 >> 172.21.15.105:0/2210461232 conn(0x7fa82c0a2080 msgr2=0x7fa82c0a2490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:21.135 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.134+0000 7fa831907640 1 -- 172.21.15.105:0/2210461232 shutdown_connections 2024-09-21T01:46:21.135 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.134+0000 7fa831907640 1 -- 172.21.15.105:0/2210461232 wait complete. 2024-09-21T01:46:21.139 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.135+0000 7fa831907640 1 Processor -- start 2024-09-21T01:46:21.139 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.135+0000 7fa831907640 1 -- start start 2024-09-21T01:46:21.139 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.135+0000 7fa831907640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa82c070c50 0x7fa82c131330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:21.140 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.135+0000 7fa831907640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa82c071060 0x7fa82c131870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:21.140 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.136+0000 7fa831907640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa82c12f770 0x7fa82c131db0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:21.140 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.136+0000 7fa831907640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7fa82c1386e0 con 0x7fa82c12f770 2024-09-21T01:46:21.140 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.136+0000 7fa831907640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7fa82c138560 con 0x7fa82c070c50 2024-09-21T01:46:21.140 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.136+0000 7fa831907640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7fa82c138860 con 0x7fa82c071060 2024-09-21T01:46:21.140 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.136+0000 7fa82affd640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa82c070c50 0x7fa82c131330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:21.140 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.136+0000 7fa823fff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa82c071060 0x7fa82c131870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:21.140 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.136+0000 7fa82affd640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa82c070c50 0x7fa82c131330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:53392/0 (socket says 172.21.15.105:53392) 2024-09-21T01:46:21.140 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.136+0000 7fa82affd640 1 -- 172.21.15.105:0/1823078047 learned_addr learned my addr 172.21.15.105:0/1823078047 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:21.141 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.136+0000 7fa82b7fe640 1 --2- 172.21.15.105:0/1823078047 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa82c12f770 0x7fa82c131db0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:21.141 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.136+0000 7fa82affd640 1 -- 172.21.15.105:0/1823078047 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa82c071060 msgr2=0x7fa82c131870 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:21.141 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.136+0000 7fa82affd640 1 --2- 172.21.15.105:0/1823078047 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa82c071060 0x7fa82c131870 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:21.141 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.136+0000 7fa82affd640 1 -- 172.21.15.105:0/1823078047 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa82c12f770 msgr2=0x7fa82c131db0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:21.141 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.136+0000 7fa82affd640 1 --2- 172.21.15.105:0/1823078047 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa82c12f770 0x7fa82c131db0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:21.141 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.136+0000 7fa82affd640 1 -- 172.21.15.105:0/1823078047 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa82c134380 con 0x7fa82c070c50 2024-09-21T01:46:21.141 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.136+0000 7fa82affd640 1 --2- 172.21.15.105:0/1823078047 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa82c070c50 0x7fa82c131330 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fa81c00e940 tx=0x7fa81c00ee00 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:21.142 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.136+0000 7fa828ff9640 1 -- 172.21.15.105:0/1823078047 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa81c00cd80 con 0x7fa82c070c50 2024-09-21T01:46:21.142 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.136+0000 7fa828ff9640 1 -- 172.21.15.105:0/1823078047 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fa81c010430 con 0x7fa82c070c50 2024-09-21T01:46:21.142 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.136+0000 7fa831907640 1 -- 172.21.15.105:0/1823078047 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa82c133dd0 con 0x7fa82c070c50 2024-09-21T01:46:21.142 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.136+0000 7fa828ff9640 1 -- 172.21.15.105:0/1823078047 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa81c010880 con 0x7fa82c070c50 2024-09-21T01:46:21.142 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.136+0000 7fa831907640 1 -- 172.21.15.105:0/1823078047 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fa82c1cbd40 con 0x7fa82c070c50 2024-09-21T01:46:21.142 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.137+0000 7fa831907640 1 -- 172.21.15.105:0/1823078047 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa82c071730 con 0x7fa82c070c50 2024-09-21T01:46:21.142 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.137+0000 7fa828ff9640 1 -- 172.21.15.105:0/1823078047 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fa81c01f020 con 0x7fa82c070c50 2024-09-21T01:46:21.142 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.138+0000 7fa828ff9640 1 -- 172.21.15.105:0/1823078047 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(24..24 src has 1..24) ==== 3027+0+0 (secure 0 0 0) 0x7fa81c069700 con 0x7fa82c070c50 2024-09-21T01:46:21.143 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.142+0000 7fa828ff9640 1 -- 172.21.15.105:0/1823078047 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fa81c009490 con 0x7fa82c070c50 2024-09-21T01:46:21.296 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.295+0000 7fa831907640 1 -- 172.21.15.105:0/1823078047 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fa82c132c80 con 0x7fa82c070c50 2024-09-21T01:46:21.296 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.296+0000 7fa828ff9640 1 -- 172.21.15.105:0/1823078047 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v24) ==== 74+0+8690 (secure 0 0 0) 0x7fa81c0687b0 con 0x7fa82c070c50 2024-09-21T01:46:21.296 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:21.297 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":24,"fsid":"355399d3-6490-4ad0-ac52-d793648ff276","created":"2024-09-21T01:44:37.282024+0000","modified":"2024-09-21T01:46:21.001569+0000","last_up_change":"2024-09-21T01:46:21.001569+0000","last_in_change":"2024-09-21T01:44:39.666350+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"reef","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T01:44:50.380956+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T01:44:54.210449+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"03218bdb-ec48-44c7-bbf5-31cb53a3cea7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":21,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6800","nonce":4053719786}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6801","nonce":4053719786}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6803","nonce":4053719786}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6802","nonce":4053719786}]},"public_addr":"172.21.15.105:6800/4053719786","cluster_addr":"172.21.15.105:6801/4053719786","heartbeat_back_addr":"172.21.15.105:6803/4053719786","heartbeat_front_addr":"172.21.15.105:6802/4053719786","state":["exists"]},{"osd":1,"uuid":"5528ff44-922a-4e44-bb69-95fd9544d8dd","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6812","nonce":1313742696}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6813","nonce":1313742696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6815","nonce":1313742696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6814","nonce":1313742696}]},"public_addr":"172.21.15.105:6812/1313742696","cluster_addr":"172.21.15.105:6813/1313742696","heartbeat_back_addr":"172.21.15.105:6815/1313742696","heartbeat_front_addr":"172.21.15.105:6814/1313742696","state":["exists"]},{"osd":2,"uuid":"df56ff35-71e9-4d46-8abb-84265eb6e1ee","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6804","nonce":2159605016}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6805","nonce":2159605016}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6807","nonce":2159605016}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6806","nonce":2159605016}]},"public_addr":"172.21.15.105:6804/2159605016","cluster_addr":"172.21.15.105:6805/2159605016","heartbeat_back_addr":"172.21.15.105:6807/2159605016","heartbeat_front_addr":"172.21.15.105:6806/2159605016","state":["exists"]},{"osd":3,"uuid":"012d9668-f1d3-4268-b0eb-15fa845ef17c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":23,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6808","nonce":2514115267}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6809","nonce":2514115267}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6811","nonce":2514115267}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6810","nonce":2514115267}]},"public_addr":"172.21.15.105:6808/2514115267","cluster_addr":"172.21.15.105:6809/2514115267","heartbeat_back_addr":"172.21.15.105:6811/2514115267","heartbeat_front_addr":"172.21.15.105:6810/2514115267","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.554795+0000","dead_epoch":21},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.869400+0000","dead_epoch":22},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.601687+0000","dead_epoch":23},{"osd":3,"down_stamp":"2024-09-21T01:46:21.001569+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.822680+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.105:0/3561680891":"2024-09-22T01:46:13.485080+0000","172.21.15.105:0/2570087590":"2024-09-22T01:46:13.485080+0000","172.21.15.105:0/3837276043":"2024-09-22T01:46:13.485080+0000","172.21.15.105:0/2810478014":"2024-09-22T01:46:13.485080+0000","172.21.15.105:6816/3933313069":"2024-09-22T01:46:13.485080+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T01:46:21.300 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.299+0000 7fa831907640 1 -- 172.21.15.105:0/1823078047 >> v1:172.21.15.105:6816/2838005468 conn(0x7fa80c0514b0 legacy=0x7fa80c0539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:21.300 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.299+0000 7fa831907640 1 -- 172.21.15.105:0/1823078047 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa82c070c50 msgr2=0x7fa82c131330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:21.300 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.299+0000 7fa831907640 1 --2- 172.21.15.105:0/1823078047 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa82c070c50 0x7fa82c131330 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7fa81c00e940 tx=0x7fa81c00ee00 comp rx=0 tx=0).stop 2024-09-21T01:46:21.300 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.299+0000 7fa831907640 1 -- 172.21.15.105:0/1823078047 shutdown_connections 2024-09-21T01:46:21.300 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.299+0000 7fa831907640 1 --2- 172.21.15.105:0/1823078047 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa82c12f770 0x7fa82c131db0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:21.300 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.299+0000 7fa831907640 1 --2- 172.21.15.105:0/1823078047 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa82c071060 0x7fa82c131870 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:21.300 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.299+0000 7fa831907640 1 --2- 172.21.15.105:0/1823078047 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa82c070c50 0x7fa82c131330 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:21.300 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.299+0000 7fa831907640 1 -- 172.21.15.105:0/1823078047 >> 172.21.15.105:0/1823078047 conn(0x7fa82c0a2080 msgr2=0x7fa82c12c190 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:21.300 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.299+0000 7fa831907640 1 -- 172.21.15.105:0/1823078047 shutdown_connections 2024-09-21T01:46:21.300 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:21.299+0000 7fa831907640 1 -- 172.21.15.105:0/1823078047 wait complete. 2024-09-21T01:46:21.438 INFO:tasks.ceph.osd.2.smithi105.stderr:2024-09-21T01:46:21.437+0000 7f0550c18640 -1 Falling back to public interface 2024-09-21T01:46:22.460 INFO:tasks.ceph.osd.3.smithi105.stderr:2024-09-21T01:46:22.459+0000 7fce129bb640 -1 Falling back to public interface 2024-09-21T01:46:23.538 INFO:tasks.ceph.osd.0.smithi105.stderr:2024-09-21T01:46:23.538+0000 7f236d20b640 -1 osd.0 20 log_to_monitors true 2024-09-21T01:46:23.604 INFO:tasks.ceph.osd.0.smithi105.stderr:2024-09-21T01:46:23.604+0000 7f23608f4640 -1 osd.0 20 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T01:46:24.250 INFO:tasks.ceph.osd.1.smithi105.stderr:2024-09-21T01:46:24.250+0000 7fb080ba8640 -1 osd.1 20 log_to_monitors true 2024-09-21T01:46:24.314 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T01:46:24.424 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.423+0000 7fbf323e4640 1 -- 172.21.15.105:0/1942991943 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbf2c072d10 msgr2=0x7fbf2c073120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:24.424 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.423+0000 7fbf323e4640 1 --2- 172.21.15.105:0/1942991943 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbf2c072d10 0x7fbf2c073120 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7fbf1c009830 tx=0x7fbf1c02a350 comp rx=0 tx=0).stop 2024-09-21T01:46:24.424 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.424+0000 7fbf323e4640 1 -- 172.21.15.105:0/1942991943 shutdown_connections 2024-09-21T01:46:24.424 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.424+0000 7fbf323e4640 1 --2- 172.21.15.105:0/1942991943 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbf2c099790 0x7fbf2c099ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:24.425 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.424+0000 7fbf323e4640 1 --2- 172.21.15.105:0/1942991943 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbf2c073660 0x7fbf2c099250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:24.425 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.424+0000 7fbf323e4640 1 --2- 172.21.15.105:0/1942991943 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbf2c072d10 0x7fbf2c073120 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:24.425 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.424+0000 7fbf323e4640 1 -- 172.21.15.105:0/1942991943 >> 172.21.15.105:0/1942991943 conn(0x7fbf2c0a01d0 msgr2=0x7fbf2c09c6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:24.425 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.424+0000 7fbf323e4640 1 -- 172.21.15.105:0/1942991943 shutdown_connections 2024-09-21T01:46:24.425 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.424+0000 7fbf323e4640 1 -- 172.21.15.105:0/1942991943 wait complete. 2024-09-21T01:46:24.427 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.425+0000 7fbf323e4640 1 Processor -- start 2024-09-21T01:46:24.427 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.425+0000 7fbf323e4640 1 -- start start 2024-09-21T01:46:24.427 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.425+0000 7fbf323e4640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbf2c072d10 0x7fbf2c130c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:24.427 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.425+0000 7fbf323e4640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbf2c073660 0x7fbf2c1311d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:24.427 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.425+0000 7fbf323e4640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbf2c099790 0x7fbf2c131710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:24.427 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.425+0000 7fbf323e4640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7fbf2c135640 con 0x7fbf2c099790 2024-09-21T01:46:24.427 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.425+0000 7fbf323e4640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7fbf2c1354c0 con 0x7fbf2c073660 2024-09-21T01:46:24.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.425+0000 7fbf323e4640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7fbf2c1357c0 con 0x7fbf2c072d10 2024-09-21T01:46:24.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.425+0000 7fbf2bfff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbf2c072d10 0x7fbf2c130c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:24.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.426+0000 7fbf2bfff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbf2c072d10 0x7fbf2c130c90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:38722/0 (socket says 172.21.15.105:38722) 2024-09-21T01:46:24.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.426+0000 7fbf2bfff640 1 -- 172.21.15.105:0/3819919878 learned_addr learned my addr 172.21.15.105:0/3819919878 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:24.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.426+0000 7fbf2b7fe640 1 --2- 172.21.15.105:0/3819919878 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbf2c073660 0x7fbf2c1311d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:24.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.426+0000 7fbf30981640 1 --2- 172.21.15.105:0/3819919878 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbf2c099790 0x7fbf2c131710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:24.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.426+0000 7fbf2bfff640 1 -- 172.21.15.105:0/3819919878 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbf2c073660 msgr2=0x7fbf2c1311d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:24.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.426+0000 7fbf2bfff640 1 --2- 172.21.15.105:0/3819919878 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbf2c073660 0x7fbf2c1311d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:24.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.426+0000 7fbf2bfff640 1 -- 172.21.15.105:0/3819919878 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbf2c099790 msgr2=0x7fbf2c131710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:24.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.426+0000 7fbf2bfff640 1 --2- 172.21.15.105:0/3819919878 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbf2c099790 0x7fbf2c131710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:24.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.426+0000 7fbf2bfff640 1 -- 172.21.15.105:0/3819919878 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fbf2c134030 con 0x7fbf2c072d10 2024-09-21T01:46:24.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.426+0000 7fbf2bfff640 1 --2- 172.21.15.105:0/3819919878 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbf2c072d10 0x7fbf2c130c90 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7fbf1c009780 tx=0x7fbf1c033af0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:24.428 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.426+0000 7fbf28ff9640 1 -- 172.21.15.105:0/3819919878 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbf1c03f020 con 0x7fbf2c072d10 2024-09-21T01:46:24.429 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.426+0000 7fbf28ff9640 1 -- 172.21.15.105:0/3819919878 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fbf1c033ce0 con 0x7fbf2c072d10 2024-09-21T01:46:24.429 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.426+0000 7fbf28ff9640 1 -- 172.21.15.105:0/3819919878 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fbf1c037040 con 0x7fbf2c072d10 2024-09-21T01:46:24.429 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.426+0000 7fbf323e4640 1 -- 172.21.15.105:0/3819919878 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fbf2c1342f0 con 0x7fbf2c072d10 2024-09-21T01:46:24.429 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.426+0000 7fbf323e4640 1 -- 172.21.15.105:0/3819919878 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fbf2c06faf0 con 0x7fbf2c072d10 2024-09-21T01:46:24.429 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.427+0000 7fbf323e4640 1 -- 172.21.15.105:0/3819919878 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fbf2c132e20 con 0x7fbf2c072d10 2024-09-21T01:46:24.429 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.427+0000 7fbf28ff9640 1 -- 172.21.15.105:0/3819919878 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fbf1c042020 con 0x7fbf2c072d10 2024-09-21T01:46:24.429 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.428+0000 7fbf28ff9640 1 -- 172.21.15.105:0/3819919878 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(25..25 src has 1..25) ==== 3027+0+0 (secure 0 0 0) 0x7fbf1c088b70 con 0x7fbf2c072d10 2024-09-21T01:46:24.432 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.432+0000 7fbf28ff9640 1 -- 172.21.15.105:0/3819919878 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fbf1c049070 con 0x7fbf2c072d10 2024-09-21T01:46:24.582 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.582+0000 7fbf323e4640 1 -- 172.21.15.105:0/3819919878 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fbf2c06a690 con 0x7fbf2c072d10 2024-09-21T01:46:24.583 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.582+0000 7fbf28ff9640 1 -- 172.21.15.105:0/3819919878 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v25) ==== 74+0+8690 (secure 0 0 0) 0x7fbf1c087c20 con 0x7fbf2c072d10 2024-09-21T01:46:24.583 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:24.583 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":25,"fsid":"355399d3-6490-4ad0-ac52-d793648ff276","created":"2024-09-21T01:44:37.282024+0000","modified":"2024-09-21T01:46:23.593047+0000","last_up_change":"2024-09-21T01:46:21.001569+0000","last_in_change":"2024-09-21T01:44:39.666350+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"reef","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T01:44:50.380956+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T01:44:54.210449+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"03218bdb-ec48-44c7-bbf5-31cb53a3cea7","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":21,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6800","nonce":4053719786}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6801","nonce":4053719786}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6803","nonce":4053719786}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6802","nonce":4053719786}]},"public_addr":"172.21.15.105:6800/4053719786","cluster_addr":"172.21.15.105:6801/4053719786","heartbeat_back_addr":"172.21.15.105:6803/4053719786","heartbeat_front_addr":"172.21.15.105:6802/4053719786","state":["exists"]},{"osd":1,"uuid":"5528ff44-922a-4e44-bb69-95fd9544d8dd","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6812","nonce":1313742696}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6813","nonce":1313742696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6815","nonce":1313742696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6814","nonce":1313742696}]},"public_addr":"172.21.15.105:6812/1313742696","cluster_addr":"172.21.15.105:6813/1313742696","heartbeat_back_addr":"172.21.15.105:6815/1313742696","heartbeat_front_addr":"172.21.15.105:6814/1313742696","state":["exists"]},{"osd":2,"uuid":"df56ff35-71e9-4d46-8abb-84265eb6e1ee","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6804","nonce":2159605016}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6805","nonce":2159605016}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6807","nonce":2159605016}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6806","nonce":2159605016}]},"public_addr":"172.21.15.105:6804/2159605016","cluster_addr":"172.21.15.105:6805/2159605016","heartbeat_back_addr":"172.21.15.105:6807/2159605016","heartbeat_front_addr":"172.21.15.105:6806/2159605016","state":["exists"]},{"osd":3,"uuid":"012d9668-f1d3-4268-b0eb-15fa845ef17c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":23,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6808","nonce":2514115267}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6809","nonce":2514115267}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6811","nonce":2514115267}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6810","nonce":2514115267}]},"public_addr":"172.21.15.105:6808/2514115267","cluster_addr":"172.21.15.105:6809/2514115267","heartbeat_back_addr":"172.21.15.105:6811/2514115267","heartbeat_front_addr":"172.21.15.105:6810/2514115267","state":["exists"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.554795+0000","dead_epoch":21},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.869400+0000","dead_epoch":22},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.601687+0000","dead_epoch":23},{"osd":3,"down_stamp":"2024-09-21T01:46:21.001569+0000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.822680+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.105:0/3561680891":"2024-09-22T01:46:13.485080+0000","172.21.15.105:0/2570087590":"2024-09-22T01:46:13.485080+0000","172.21.15.105:0/3837276043":"2024-09-22T01:46:13.485080+0000","172.21.15.105:0/2810478014":"2024-09-22T01:46:13.485080+0000","172.21.15.105:6816/3933313069":"2024-09-22T01:46:13.485080+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T01:46:24.585 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.585+0000 7fbf323e4640 1 -- 172.21.15.105:0/3819919878 >> v1:172.21.15.105:6816/2838005468 conn(0x7fbf180514b0 legacy=0x7fbf180539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:24.585 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.585+0000 7fbf323e4640 1 -- 172.21.15.105:0/3819919878 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbf2c072d10 msgr2=0x7fbf2c130c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:24.585 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.585+0000 7fbf323e4640 1 --2- 172.21.15.105:0/3819919878 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbf2c072d10 0x7fbf2c130c90 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7fbf1c009780 tx=0x7fbf1c033af0 comp rx=0 tx=0).stop 2024-09-21T01:46:24.586 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.585+0000 7fbf323e4640 1 -- 172.21.15.105:0/3819919878 shutdown_connections 2024-09-21T01:46:24.586 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.585+0000 7fbf323e4640 1 --2- 172.21.15.105:0/3819919878 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fbf2c099790 0x7fbf2c131710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:24.586 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.585+0000 7fbf323e4640 1 --2- 172.21.15.105:0/3819919878 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fbf2c073660 0x7fbf2c1311d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:24.586 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.585+0000 7fbf323e4640 1 --2- 172.21.15.105:0/3819919878 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fbf2c072d10 0x7fbf2c130c90 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:24.586 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.585+0000 7fbf323e4640 1 -- 172.21.15.105:0/3819919878 >> 172.21.15.105:0/3819919878 conn(0x7fbf2c0a01d0 msgr2=0x7fbf2c126c70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:24.586 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.585+0000 7fbf323e4640 1 -- 172.21.15.105:0/3819919878 shutdown_connections 2024-09-21T01:46:24.586 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:24.585+0000 7fbf323e4640 1 -- 172.21.15.105:0/3819919878 wait complete. 2024-09-21T01:46:24.608 INFO:tasks.ceph.osd.1.smithi105.stderr:2024-09-21T01:46:24.607+0000 7fb07492d640 -1 osd.1 20 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T01:46:25.997 INFO:tasks.ceph.osd.2.smithi105.stderr:2024-09-21T01:46:25.997+0000 7f0550c18640 -1 osd.2 22 log_to_monitors true 2024-09-21T01:46:26.260 INFO:tasks.ceph.osd.3.smithi105.stderr:2024-09-21T01:46:26.260+0000 7fce129bb640 -1 osd.3 23 log_to_monitors true 2024-09-21T01:46:26.628 INFO:tasks.ceph.osd.3.smithi105.stderr:2024-09-21T01:46:26.628+0000 7fce06740640 -1 osd.3 23 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T01:46:26.628 INFO:tasks.ceph.osd.2.smithi105.stderr:2024-09-21T01:46:26.628+0000 7f05443d6640 -1 osd.2 22 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T01:46:27.597 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T01:46:27.725 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.725+0000 7f08b9951640 1 -- 172.21.15.105:0/829922136 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f08b412fd90 msgr2=0x7f08b4137a20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:27.726 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.725+0000 7f08b9951640 1 --2- 172.21.15.105:0/829922136 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f08b412fd90 0x7f08b4137a20 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f08ac009860 tx=0x7f08ac02a420 comp rx=0 tx=0).stop 2024-09-21T01:46:27.726 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.725+0000 7f08b9951640 1 -- 172.21.15.105:0/829922136 shutdown_connections 2024-09-21T01:46:27.726 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.725+0000 7f08b9951640 1 --2- 172.21.15.105:0/829922136 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f08b412fd90 0x7f08b4137a20 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:27.726 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.725+0000 7f08b9951640 1 --2- 172.21.15.105:0/829922136 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f08b40717b0 0x7f08b412c570 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:27.726 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.725+0000 7f08b9951640 1 -- 172.21.15.105:0/829922136 >> 172.21.15.105:0/829922136 conn(0x7f08b40a2180 msgr2=0x7f08b40a2590 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:27.726 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.725+0000 7f08b9951640 1 -- 172.21.15.105:0/829922136 shutdown_connections 2024-09-21T01:46:27.726 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.725+0000 7f08b9951640 1 -- 172.21.15.105:0/829922136 wait complete. 2024-09-21T01:46:27.727 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.726+0000 7f08b9951640 1 Processor -- start 2024-09-21T01:46:27.727 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.726+0000 7f08b9951640 1 -- start start 2024-09-21T01:46:27.728 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.727+0000 7f08b9951640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f08b40717b0 0x7f08b4131600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:27.728 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.727+0000 7f08b9951640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f08b412cab0 0x7f08b4131b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:27.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.727+0000 7f08b9951640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f08b412fd90 0x7f08b4132080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:27.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.727+0000 7f08b9951640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f08b41386e0 con 0x7f08b412cab0 2024-09-21T01:46:27.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.727+0000 7f08b9951640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f08b4138560 con 0x7f08b412fd90 2024-09-21T01:46:27.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.727+0000 7f08b9951640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f08b4138860 con 0x7f08b40717b0 2024-09-21T01:46:27.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.727+0000 7f08b2ffd640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f08b40717b0 0x7f08b4131600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:27.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.727+0000 7f08b2ffd640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f08b40717b0 0x7f08b4131600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:38738/0 (socket says 172.21.15.105:38738) 2024-09-21T01:46:27.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.727+0000 7f08b2ffd640 1 -- 172.21.15.105:0/3082580174 learned_addr learned my addr 172.21.15.105:0/3082580174 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:27.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.727+0000 7f08b27fc640 1 --2- 172.21.15.105:0/3082580174 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f08b412cab0 0x7f08b4131b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:27.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.727+0000 7f08b37fe640 1 --2- 172.21.15.105:0/3082580174 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f08b412fd90 0x7f08b4132080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:27.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.727+0000 7f08b2ffd640 1 -- 172.21.15.105:0/3082580174 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f08b412fd90 msgr2=0x7f08b4132080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:27.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.727+0000 7f08b2ffd640 1 --2- 172.21.15.105:0/3082580174 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f08b412fd90 0x7f08b4132080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:27.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.727+0000 7f08b2ffd640 1 -- 172.21.15.105:0/3082580174 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f08b412cab0 msgr2=0x7f08b4131b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:27.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.727+0000 7f08b2ffd640 1 --2- 172.21.15.105:0/3082580174 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f08b412cab0 0x7f08b4131b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:27.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.727+0000 7f08b2ffd640 1 -- 172.21.15.105:0/3082580174 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f08b41377b0 con 0x7f08b40717b0 2024-09-21T01:46:27.730 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.728+0000 7f08b2ffd640 1 --2- 172.21.15.105:0/3082580174 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f08b40717b0 0x7f08b4131600 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f08a400ef90 tx=0x7f08a400c800 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:27.730 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.728+0000 7f089bfff640 1 -- 172.21.15.105:0/3082580174 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f08a4003970 con 0x7f08b40717b0 2024-09-21T01:46:27.730 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.728+0000 7f089bfff640 1 -- 172.21.15.105:0/3082580174 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f08a4003c00 con 0x7f08b40717b0 2024-09-21T01:46:27.730 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.728+0000 7f08b9951640 1 -- 172.21.15.105:0/3082580174 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f08b4134690 con 0x7f08b40717b0 2024-09-21T01:46:27.730 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.728+0000 7f089bfff640 1 -- 172.21.15.105:0/3082580174 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f08a4004020 con 0x7f08b40717b0 2024-09-21T01:46:27.730 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.728+0000 7f08b9951640 1 -- 172.21.15.105:0/3082580174 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f08b41cbb60 con 0x7f08b40717b0 2024-09-21T01:46:27.730 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.729+0000 7f08b9951640 1 -- 172.21.15.105:0/3082580174 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f08b412c570 con 0x7f08b40717b0 2024-09-21T01:46:27.731 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.729+0000 7f089bfff640 1 -- 172.21.15.105:0/3082580174 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f08a401d020 con 0x7f08b40717b0 2024-09-21T01:46:27.731 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.729+0000 7f089bfff640 1 -- 172.21.15.105:0/3082580174 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(29..29 src has 1..29) ==== 3027+0+0 (secure 0 0 0) 0x7f08a40694e0 con 0x7f08b40717b0 2024-09-21T01:46:27.733 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.733+0000 7f089bfff640 1 -- 172.21.15.105:0/3082580174 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f08a4068590 con 0x7f08b40717b0 2024-09-21T01:46:27.884 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.883+0000 7f08b9951640 1 -- 172.21.15.105:0/3082580174 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f08b4129540 con 0x7f08b40717b0 2024-09-21T01:46:27.884 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.884+0000 7f089bfff640 1 -- 172.21.15.105:0/3082580174 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v29) ==== 74+0+9195 (secure 0 0 0) 0x7f08a4009780 con 0x7f08b40717b0 2024-09-21T01:46:27.884 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:27.885 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":29,"fsid":"355399d3-6490-4ad0-ac52-d793648ff276","created":"2024-09-21T01:44:37.282024+0000","modified":"2024-09-21T01:46:27.619214+0000","last_up_change":"2024-09-21T01:46:27.619214+0000","last_in_change":"2024-09-21T01:44:39.666350+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":7,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"reef","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T01:44:50.380956+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T01:44:54.210449+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"03218bdb-ec48-44c7-bbf5-31cb53a3cea7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":20,"up_from":26,"up_thru":26,"down_at":21,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6800","nonce":249246859}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6801","nonce":249246859}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6803","nonce":249246859}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6802","nonce":249246859}]},"public_addr":"172.21.15.105:6800/249246859","cluster_addr":"172.21.15.105:6801/249246859","heartbeat_back_addr":"172.21.15.105:6803/249246859","heartbeat_front_addr":"172.21.15.105:6802/249246859","state":["exists","up"]},{"osd":1,"uuid":"5528ff44-922a-4e44-bb69-95fd9544d8dd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":20,"up_from":27,"up_thru":27,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6804","nonce":1896987326}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6805","nonce":1896987326}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6807","nonce":1896987326}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6806","nonce":1896987326}]},"public_addr":"172.21.15.105:6804/1896987326","cluster_addr":"172.21.15.105:6805/1896987326","heartbeat_back_addr":"172.21.15.105:6807/1896987326","heartbeat_front_addr":"172.21.15.105:6806/1896987326","state":["exists","up"]},{"osd":2,"uuid":"df56ff35-71e9-4d46-8abb-84265eb6e1ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":22,"up_from":29,"up_thru":14,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6808","nonce":2933198036}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6809","nonce":2933198036}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6811","nonce":2933198036}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6810","nonce":2933198036}]},"public_addr":"172.21.15.105:6808/2933198036","cluster_addr":"172.21.15.105:6809/2933198036","heartbeat_back_addr":"172.21.15.105:6811/2933198036","heartbeat_front_addr":"172.21.15.105:6810/2933198036","state":["exists","up"]},{"osd":3,"uuid":"012d9668-f1d3-4268-b0eb-15fa845ef17c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":23,"up_from":29,"up_thru":23,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6812","nonce":568702146}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6813","nonce":568702146}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6815","nonce":568702146}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6814","nonce":568702146}]},"public_addr":"172.21.15.105:6812/568702146","cluster_addr":"172.21.15.105:6813/568702146","heartbeat_back_addr":"172.21.15.105:6815/568702146","heartbeat_front_addr":"172.21.15.105:6814/568702146","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.554795+0000","dead_epoch":21},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.869400+0000","dead_epoch":22},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.601687+0000","dead_epoch":23},{"osd":3,"down_stamp":"2024-09-21T01:46:21.001569+0000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.822680+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.105:0/3561680891":"2024-09-22T01:46:13.485080+0000","172.21.15.105:0/2570087590":"2024-09-22T01:46:13.485080+0000","172.21.15.105:0/3837276043":"2024-09-22T01:46:13.485080+0000","172.21.15.105:0/2810478014":"2024-09-22T01:46:13.485080+0000","172.21.15.105:6816/3933313069":"2024-09-22T01:46:13.485080+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T01:46:27.887 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.886+0000 7f08b9951640 1 -- 172.21.15.105:0/3082580174 >> v1:172.21.15.105:6816/2838005468 conn(0x7f08a00514b0 legacy=0x7f08a00539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:27.887 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.886+0000 7f08b9951640 1 -- 172.21.15.105:0/3082580174 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f08b40717b0 msgr2=0x7f08b4131600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:27.887 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.886+0000 7f08b9951640 1 --2- 172.21.15.105:0/3082580174 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f08b40717b0 0x7f08b4131600 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f08a400ef90 tx=0x7f08a400c800 comp rx=0 tx=0).stop 2024-09-21T01:46:27.887 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.886+0000 7f08b9951640 1 -- 172.21.15.105:0/3082580174 shutdown_connections 2024-09-21T01:46:27.887 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.886+0000 7f08b9951640 1 --2- 172.21.15.105:0/3082580174 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f08b412fd90 0x7f08b4132080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:27.887 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.886+0000 7f08b9951640 1 --2- 172.21.15.105:0/3082580174 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f08b412cab0 0x7f08b4131b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:27.887 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.886+0000 7f08b9951640 1 --2- 172.21.15.105:0/3082580174 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f08b40717b0 0x7f08b4131600 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:27.887 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.886+0000 7f08b9951640 1 -- 172.21.15.105:0/3082580174 >> 172.21.15.105:0/3082580174 conn(0x7f08b40a2180 msgr2=0x7f08b412f470 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:27.887 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.886+0000 7f08b9951640 1 -- 172.21.15.105:0/3082580174 shutdown_connections 2024-09-21T01:46:27.887 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:27.886+0000 7f08b9951640 1 -- 172.21.15.105:0/3082580174 wait complete. 2024-09-21T01:46:27.898 INFO:tasks.ceph.ceph_manager.ceph:all up! 2024-09-21T01:46:27.898 INFO:teuthology.run_tasks:Running task exec... 2024-09-21T01:46:27.909 INFO:teuthology.task.exec:Executing custom commands... 2024-09-21T01:46:27.909 INFO:teuthology.task.exec:Running commands on role mon.a host ubuntu@smithi105.front.sepia.ceph.com 2024-09-21T01:46:27.909 DEBUG:teuthology.orchestra.run.smithi105:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph versions' 2024-09-21T01:46:28.015 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.014+0000 7f07b276a640 1 -- 172.21.15.105:0/4203822485 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f07ac129360 msgr2=0x7f07ac129770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:28.015 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.014+0000 7f07b276a640 1 --2- 172.21.15.105:0/4203822485 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f07ac129360 0x7f07ac129770 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f079c009830 tx=0x7f079c02a480 comp rx=0 tx=0).stop 2024-09-21T01:46:28.015 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.014+0000 7f07b276a640 1 -- 172.21.15.105:0/4203822485 shutdown_connections 2024-09-21T01:46:28.015 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.014+0000 7f07b276a640 1 --2- 172.21.15.105:0/4203822485 >> v2:172.21.15.105:3300/0 conn(0x7f07ac0939f0 0x7f07ac093e00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.015 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.014+0000 7f07b276a640 1 --2- 172.21.15.105:0/4203822485 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f07ac129360 0x7f07ac129770 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.015 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.014+0000 7f07b276a640 1 -- 172.21.15.105:0/4203822485 >> 172.21.15.105:0/4203822485 conn(0x7f07ac09c770 msgr2=0x7f07ac09e240 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:28.015 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.014+0000 7f07b276a640 1 -- 172.21.15.105:0/4203822485 shutdown_connections 2024-09-21T01:46:28.015 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.014+0000 7f07b276a640 1 -- 172.21.15.105:0/4203822485 wait complete. 2024-09-21T01:46:28.016 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.015+0000 7f07b276a640 1 Processor -- start 2024-09-21T01:46:28.016 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.016+0000 7f07b276a640 1 -- start start 2024-09-21T01:46:28.017 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.016+0000 7f07b276a640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f07ac0939f0 0x7f07ac133d10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:28.017 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.016+0000 7f07b276a640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f07ac129360 0x7f07ac134250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:28.018 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.016+0000 7f07b276a640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f07ac129cb0 0x7f07ac0733b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:28.018 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.016+0000 7f07b276a640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f07ac135400 con 0x7f07ac0939f0 2024-09-21T01:46:28.018 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.016+0000 7f07b276a640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f07ac135280 con 0x7f07ac129360 2024-09-21T01:46:28.018 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.016+0000 7f07b276a640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f07ac135580 con 0x7f07ac129cb0 2024-09-21T01:46:28.018 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.016+0000 7f07abfff640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f07ac0939f0 0x7f07ac133d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:28.018 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.016+0000 7f07ab7fe640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f07ac129360 0x7f07ac134250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:28.018 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.016+0000 7f07abfff640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f07ac0939f0 0x7f07ac133d10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:34844/0 (socket says 172.21.15.105:34844) 2024-09-21T01:46:28.018 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.016+0000 7f07abfff640 1 -- 172.21.15.105:0/1970298419 learned_addr learned my addr 172.21.15.105:0/1970298419 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:28.018 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.016+0000 7f07b0d07640 1 --2- 172.21.15.105:0/1970298419 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f07ac129cb0 0x7f07ac0733b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:28.018 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.017+0000 7f07ab7fe640 1 -- 172.21.15.105:0/1970298419 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f07ac129cb0 msgr2=0x7f07ac0733b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:28.018 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.017+0000 7f07ab7fe640 1 --2- 172.21.15.105:0/1970298419 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f07ac129cb0 0x7f07ac0733b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.018 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.017+0000 7f07ab7fe640 1 -- 172.21.15.105:0/1970298419 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f07ac0939f0 msgr2=0x7f07ac133d10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:28.018 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.017+0000 7f07ab7fe640 1 --2- 172.21.15.105:0/1970298419 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f07ac0939f0 0x7f07ac133d10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.018 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.017+0000 7f07ab7fe640 1 -- 172.21.15.105:0/1970298419 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f07ac123cd0 con 0x7f07ac129360 2024-09-21T01:46:28.019 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.017+0000 7f07abfff640 1 --2- 172.21.15.105:0/1970298419 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f07ac0939f0 0x7f07ac133d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:28.019 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.017+0000 7f07ab7fe640 1 --2- 172.21.15.105:0/1970298419 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f07ac129360 0x7f07ac134250 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f07ac075420 tx=0x7f07a000c760 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:28.019 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.017+0000 7f07a8ff9640 1 -- 172.21.15.105:0/1970298419 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f07a0017070 con 0x7f07ac129360 2024-09-21T01:46:28.019 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.017+0000 7f07a8ff9640 1 -- 172.21.15.105:0/1970298419 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f07a00035f0 con 0x7f07ac129360 2024-09-21T01:46:28.019 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.017+0000 7f07b276a640 1 -- 172.21.15.105:0/1970298419 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f07ac134790 con 0x7f07ac129360 2024-09-21T01:46:28.019 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.017+0000 7f07a8ff9640 1 -- 172.21.15.105:0/1970298419 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f07a0003a40 con 0x7f07ac129360 2024-09-21T01:46:28.019 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.017+0000 7f07b276a640 1 -- 172.21.15.105:0/1970298419 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f07ac073ad0 con 0x7f07ac129360 2024-09-21T01:46:28.019 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.018+0000 7f07b276a640 1 -- 172.21.15.105:0/1970298419 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f07ac093e00 con 0x7f07ac129360 2024-09-21T01:46:28.019 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.018+0000 7f07a8ff9640 1 -- 172.21.15.105:0/1970298419 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f07a001a070 con 0x7f07ac129360 2024-09-21T01:46:28.020 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.019+0000 7f07a8ff9640 1 -- 172.21.15.105:0/1970298419 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(29..29 src has 1..29) ==== 3027+0+0 (secure 0 0 0) 0x7f07a00680e0 con 0x7f07ac129360 2024-09-21T01:46:28.023 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.022+0000 7f07a8ff9640 1 -- 172.21.15.105:0/1970298419 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f07a0067190 con 0x7f07ac129360 2024-09-21T01:46:28.230 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.229+0000 7f07b276a640 1 -- 172.21.15.105:0/1970298419 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "versions"} v 0) -- 0x7f07ac069750 con 0x7f07ac129360 2024-09-21T01:46:28.230 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.230+0000 7f07a8ff9640 1 -- 172.21.15.105:0/1970298419 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{"prefix": "versions"}]=0 v0) ==== 56+0+499 (secure 0 0 0) 0x7f07a0066cd0 con 0x7f07ac129360 2024-09-21T01:46:28.230 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-21T01:46:28.231 INFO:teuthology.orchestra.run.smithi105.stdout: "mon": { 2024-09-21T01:46:28.231 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.3.0-5120-g18432218 (1843221895adccfb3b692060251dd6a0183bc410) squid (dev)": 3 2024-09-21T01:46:28.231 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.231 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": { 2024-09-21T01:46:28.231 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.3.0-5120-g18432218 (1843221895adccfb3b692060251dd6a0183bc410) squid (dev)": 2 2024-09-21T01:46:28.231 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.231 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": { 2024-09-21T01:46:28.231 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.3.0-5120-g18432218 (1843221895adccfb3b692060251dd6a0183bc410) squid (dev)": 4 2024-09-21T01:46:28.231 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.231 INFO:teuthology.orchestra.run.smithi105.stdout: "overall": { 2024-09-21T01:46:28.231 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph version 19.3.0-5120-g18432218 (1843221895adccfb3b692060251dd6a0183bc410) squid (dev)": 9 2024-09-21T01:46:28.231 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.231 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-21T01:46:28.232 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.232+0000 7f07b276a640 1 -- 172.21.15.105:0/1970298419 >> v1:172.21.15.105:6816/2838005468 conn(0x7f07980514e0 legacy=0x7f07980539f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:28.232 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.232+0000 7f07b276a640 1 -- 172.21.15.105:0/1970298419 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f07ac129360 msgr2=0x7f07ac134250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:28.232 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.232+0000 7f07b276a640 1 --2- 172.21.15.105:0/1970298419 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f07ac129360 0x7f07ac134250 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f07ac075420 tx=0x7f07a000c760 comp rx=0 tx=0).stop 2024-09-21T01:46:28.233 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.232+0000 7f07b276a640 1 -- 172.21.15.105:0/1970298419 shutdown_connections 2024-09-21T01:46:28.233 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.232+0000 7f07b276a640 1 --2- 172.21.15.105:0/1970298419 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f07ac129cb0 0x7f07ac0733b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.233 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.232+0000 7f07b276a640 1 --2- 172.21.15.105:0/1970298419 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f07ac129360 0x7f07ac134250 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.233 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.232+0000 7f07b276a640 1 --2- 172.21.15.105:0/1970298419 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f07ac0939f0 0x7f07ac133d10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.233 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.232+0000 7f07b276a640 1 -- 172.21.15.105:0/1970298419 >> 172.21.15.105:0/1970298419 conn(0x7f07ac09c770 msgr2=0x7f07ac0921d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:28.233 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.232+0000 7f07b276a640 1 -- 172.21.15.105:0/1970298419 shutdown_connections 2024-09-21T01:46:28.233 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.232+0000 7f07b276a640 1 -- 172.21.15.105:0/1970298419 wait complete. 2024-09-21T01:46:28.243 DEBUG:teuthology.orchestra.run.smithi105:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph osd dump -f json-pretty' 2024-09-21T01:46:28.350 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.349+0000 7f0c5cd07640 1 -- 172.21.15.105:0/2669162016 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0c58096550 msgr2=0x7f0c58096960 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:28.350 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.349+0000 7f0c5cd07640 1 --2- 172.21.15.105:0/2669162016 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0c58096550 0x7f0c58096960 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f0c50005fd0 tx=0x7f0c5002a770 comp rx=0 tx=0).stop 2024-09-21T01:46:28.350 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.349+0000 7f0c5cd07640 1 -- 172.21.15.105:0/2669162016 shutdown_connections 2024-09-21T01:46:28.350 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.349+0000 7f0c5cd07640 1 --2- 172.21.15.105:0/2669162016 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0c58096550 0x7f0c58096960 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.350 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.349+0000 7f0c5cd07640 1 --2- 172.21.15.105:0/2669162016 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0c5812dd80 0x7f0c58096010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.350 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.350+0000 7f0c5cd07640 1 --2- 172.21.15.105:0/2669162016 >> v2:172.21.15.105:3300/0 conn(0x7f0c5812d430 0x7f0c5812d840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.351 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.350+0000 7f0c5cd07640 1 -- 172.21.15.105:0/2669162016 >> 172.21.15.105:0/2669162016 conn(0x7f0c5809e1a0 msgr2=0x7f0c5809fbb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:28.351 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.350+0000 7f0c5cd07640 1 -- 172.21.15.105:0/2669162016 shutdown_connections 2024-09-21T01:46:28.351 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.350+0000 7f0c5cd07640 1 -- 172.21.15.105:0/2669162016 wait complete. 2024-09-21T01:46:28.351 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.351+0000 7f0c5cd07640 1 Processor -- start 2024-09-21T01:46:28.351 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.351+0000 7f0c5cd07640 1 -- start start 2024-09-21T01:46:28.353 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c5cd07640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0c58096550 0x7f0c5812e9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:28.353 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c5cd07640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c5812d430 0x7f0c5812eee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:28.353 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c5cd07640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0c5812dd80 0x7f0c5812f420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:28.353 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c5cd07640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f0c581352e0 con 0x7f0c5812d430 2024-09-21T01:46:28.353 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c5cd07640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f0c58135160 con 0x7f0c5812dd80 2024-09-21T01:46:28.353 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c5cd07640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f0c58135460 con 0x7f0c58096550 2024-09-21T01:46:28.353 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c4dd9b640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c5812d430 0x7f0c5812eee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:28.353 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c5659c640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0c58096550 0x7f0c5812e9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:28.354 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c56d9d640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0c5812dd80 0x7f0c5812f420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:28.354 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c4dd9b640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c5812d430 0x7f0c5812eee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:34862/0 (socket says 172.21.15.105:34862) 2024-09-21T01:46:28.354 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c5659c640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0c58096550 0x7f0c5812e9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:38756/0 (socket says 172.21.15.105:38756) 2024-09-21T01:46:28.354 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c4dd9b640 1 -- 172.21.15.105:0/3246887750 learned_addr learned my addr 172.21.15.105:0/3246887750 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:28.354 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c5659c640 1 -- 172.21.15.105:0/3246887750 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0c5812dd80 msgr2=0x7f0c5812f420 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:28.354 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c5659c640 1 --2- 172.21.15.105:0/3246887750 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0c5812dd80 0x7f0c5812f420 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.354 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c5659c640 1 -- 172.21.15.105:0/3246887750 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c5812d430 msgr2=0x7f0c5812eee0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:28.354 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c5659c640 1 --2- 172.21.15.105:0/3246887750 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c5812d430 0x7f0c5812eee0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.354 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c5659c640 1 -- 172.21.15.105:0/3246887750 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0c5809ce30 con 0x7f0c58096550 2024-09-21T01:46:28.354 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c56d9d640 1 --2- 172.21.15.105:0/3246887750 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0c5812dd80 0x7f0c5812f420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:28.354 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c4dd9b640 1 --2- 172.21.15.105:0/3246887750 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c5812d430 0x7f0c5812eee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:28.354 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c5659c640 1 --2- 172.21.15.105:0/3246887750 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0c58096550 0x7f0c5812e9a0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f0c48009d10 tx=0x7f0c4800b690 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:28.354 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c4f7fe640 1 -- 172.21.15.105:0/3246887750 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0c48004020 con 0x7f0c58096550 2024-09-21T01:46:28.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c4f7fe640 1 -- 172.21.15.105:0/3246887750 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f0c480041c0 con 0x7f0c58096550 2024-09-21T01:46:28.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c5cd07640 1 -- 172.21.15.105:0/3246887750 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0c58130080 con 0x7f0c58096550 2024-09-21T01:46:28.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c4f7fe640 1 -- 172.21.15.105:0/3246887750 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0c480035f0 con 0x7f0c58096550 2024-09-21T01:46:28.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.352+0000 7f0c5cd07640 1 -- 172.21.15.105:0/3246887750 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f0c581c3920 con 0x7f0c58096550 2024-09-21T01:46:28.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.353+0000 7f0c5cd07640 1 -- 172.21.15.105:0/3246887750 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0c5812fab0 con 0x7f0c58096550 2024-09-21T01:46:28.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.354+0000 7f0c4f7fe640 1 -- 172.21.15.105:0/3246887750 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f0c48009310 con 0x7f0c58096550 2024-09-21T01:46:28.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.354+0000 7f0c4f7fe640 1 -- 172.21.15.105:0/3246887750 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(29..29 src has 1..29) ==== 3027+0+0 (secure 0 0 0) 0x7f0c4806cef0 con 0x7f0c58096550 2024-09-21T01:46:28.358 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.358+0000 7f0c4f7fe640 1 -- 172.21.15.105:0/3246887750 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f0c5812fab0 con 0x7f0c58096550 2024-09-21T01:46:28.506 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.506+0000 7f0c5cd07640 1 -- 172.21.15.105:0/3246887750 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd dump", "format": "json-pretty"} v 0) -- 0x7f0c58130840 con 0x7f0c58096550 2024-09-21T01:46:28.507 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.506+0000 7f0c4f7fe640 1 -- 172.21.15.105:0/3246887750 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json-pretty"}]=0 v29) ==== 81+0+16859 (secure 0 0 0) 0x7f0c4806bfa0 con 0x7f0c58096550 2024-09-21T01:46:28.509 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:28.509 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-21T01:46:28.509 INFO:teuthology.orchestra.run.smithi105.stdout: "epoch": 29, 2024-09-21T01:46:28.509 INFO:teuthology.orchestra.run.smithi105.stdout: "fsid": "355399d3-6490-4ad0-ac52-d793648ff276", 2024-09-21T01:46:28.510 INFO:teuthology.orchestra.run.smithi105.stdout: "created": "2024-09-21T01:44:37.282024+0000", 2024-09-21T01:46:28.510 INFO:teuthology.orchestra.run.smithi105.stdout: "modified": "2024-09-21T01:46:27.619214+0000", 2024-09-21T01:46:28.510 INFO:teuthology.orchestra.run.smithi105.stdout: "last_up_change": "2024-09-21T01:46:27.619214+0000", 2024-09-21T01:46:28.510 INFO:teuthology.orchestra.run.smithi105.stdout: "last_in_change": "2024-09-21T01:44:39.666350+0000", 2024-09-21T01:46:28.510 INFO:teuthology.orchestra.run.smithi105.stdout: "flags": "sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit", 2024-09-21T01:46:28.510 INFO:teuthology.orchestra.run.smithi105.stdout: "flags_num": 5799936, 2024-09-21T01:46:28.510 INFO:teuthology.orchestra.run.smithi105.stdout: "flags_set": [ 2024-09-21T01:46:28.510 INFO:teuthology.orchestra.run.smithi105.stdout: "pglog_hardlimit", 2024-09-21T01:46:28.510 INFO:teuthology.orchestra.run.smithi105.stdout: "purged_snapdirs", 2024-09-21T01:46:28.510 INFO:teuthology.orchestra.run.smithi105.stdout: "recovery_deletes", 2024-09-21T01:46:28.510 INFO:teuthology.orchestra.run.smithi105.stdout: "sortbitwise" 2024-09-21T01:46:28.510 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-21T01:46:28.511 INFO:teuthology.orchestra.run.smithi105.stdout: "crush_version": 7, 2024-09-21T01:46:28.511 INFO:teuthology.orchestra.run.smithi105.stdout: "full_ratio": 0.94999998807907104, 2024-09-21T01:46:28.511 INFO:teuthology.orchestra.run.smithi105.stdout: "backfillfull_ratio": 0.89999997615814209, 2024-09-21T01:46:28.511 INFO:teuthology.orchestra.run.smithi105.stdout: "nearfull_ratio": 0.85000002384185791, 2024-09-21T01:46:28.511 INFO:teuthology.orchestra.run.smithi105.stdout: "cluster_snapshot": "", 2024-09-21T01:46:28.511 INFO:teuthology.orchestra.run.smithi105.stdout: "pool_max": 2, 2024-09-21T01:46:28.511 INFO:teuthology.orchestra.run.smithi105.stdout: "max_osd": 4, 2024-09-21T01:46:28.511 INFO:teuthology.orchestra.run.smithi105.stdout: "require_min_compat_client": "reef", 2024-09-21T01:46:28.511 INFO:teuthology.orchestra.run.smithi105.stdout: "min_compat_client": "jewel", 2024-09-21T01:46:28.511 INFO:teuthology.orchestra.run.smithi105.stdout: "require_osd_release": "reef", 2024-09-21T01:46:28.511 INFO:teuthology.orchestra.run.smithi105.stdout: "allow_crimson": false, 2024-09-21T01:46:28.511 INFO:teuthology.orchestra.run.smithi105.stdout: "pools": [ 2024-09-21T01:46:28.511 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.512 INFO:teuthology.orchestra.run.smithi105.stdout: "pool": 1, 2024-09-21T01:46:28.512 INFO:teuthology.orchestra.run.smithi105.stdout: "pool_name": ".mgr", 2024-09-21T01:46:28.512 INFO:teuthology.orchestra.run.smithi105.stdout: "create_time": "2024-09-21T01:44:50.380956+0000", 2024-09-21T01:46:28.512 INFO:teuthology.orchestra.run.smithi105.stdout: "flags": 1, 2024-09-21T01:46:28.512 INFO:teuthology.orchestra.run.smithi105.stdout: "flags_names": "hashpspool", 2024-09-21T01:46:28.512 INFO:teuthology.orchestra.run.smithi105.stdout: "type": 1, 2024-09-21T01:46:28.512 INFO:teuthology.orchestra.run.smithi105.stdout: "size": 2, 2024-09-21T01:46:28.512 INFO:teuthology.orchestra.run.smithi105.stdout: "min_size": 1, 2024-09-21T01:46:28.512 INFO:teuthology.orchestra.run.smithi105.stdout: "crush_rule": 0, 2024-09-21T01:46:28.512 INFO:teuthology.orchestra.run.smithi105.stdout: "peering_crush_bucket_count": 0, 2024-09-21T01:46:28.512 INFO:teuthology.orchestra.run.smithi105.stdout: "peering_crush_bucket_target": 0, 2024-09-21T01:46:28.512 INFO:teuthology.orchestra.run.smithi105.stdout: "peering_crush_bucket_barrier": 0, 2024-09-21T01:46:28.512 INFO:teuthology.orchestra.run.smithi105.stdout: "peering_crush_bucket_mandatory_member": 2147483647, 2024-09-21T01:46:28.512 INFO:teuthology.orchestra.run.smithi105.stdout: "is_stretch_pool": false, 2024-09-21T01:46:28.513 INFO:teuthology.orchestra.run.smithi105.stdout: "object_hash": 2, 2024-09-21T01:46:28.513 INFO:teuthology.orchestra.run.smithi105.stdout: "pg_autoscale_mode": "off", 2024-09-21T01:46:28.513 INFO:teuthology.orchestra.run.smithi105.stdout: "pg_num": 1, 2024-09-21T01:46:28.513 INFO:teuthology.orchestra.run.smithi105.stdout: "pg_placement_num": 1, 2024-09-21T01:46:28.513 INFO:teuthology.orchestra.run.smithi105.stdout: "pg_placement_num_target": 1, 2024-09-21T01:46:28.513 INFO:teuthology.orchestra.run.smithi105.stdout: "pg_num_target": 1, 2024-09-21T01:46:28.513 INFO:teuthology.orchestra.run.smithi105.stdout: "pg_num_pending": 1, 2024-09-21T01:46:28.513 INFO:teuthology.orchestra.run.smithi105.stdout: "last_pg_merge_meta": { 2024-09-21T01:46:28.513 INFO:teuthology.orchestra.run.smithi105.stdout: "source_pgid": "0.0", 2024-09-21T01:46:28.513 INFO:teuthology.orchestra.run.smithi105.stdout: "ready_epoch": 0, 2024-09-21T01:46:28.513 INFO:teuthology.orchestra.run.smithi105.stdout: "last_epoch_started": 0, 2024-09-21T01:46:28.513 INFO:teuthology.orchestra.run.smithi105.stdout: "last_epoch_clean": 0, 2024-09-21T01:46:28.513 INFO:teuthology.orchestra.run.smithi105.stdout: "source_version": "0'0", 2024-09-21T01:46:28.513 INFO:teuthology.orchestra.run.smithi105.stdout: "target_version": "0'0" 2024-09-21T01:46:28.514 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.514 INFO:teuthology.orchestra.run.smithi105.stdout: "last_change": "13", 2024-09-21T01:46:28.514 INFO:teuthology.orchestra.run.smithi105.stdout: "last_force_op_resend": "0", 2024-09-21T01:46:28.514 INFO:teuthology.orchestra.run.smithi105.stdout: "last_force_op_resend_prenautilus": "0", 2024-09-21T01:46:28.514 INFO:teuthology.orchestra.run.smithi105.stdout: "last_force_op_resend_preluminous": "0", 2024-09-21T01:46:28.514 INFO:teuthology.orchestra.run.smithi105.stdout: "auid": 0, 2024-09-21T01:46:28.514 INFO:teuthology.orchestra.run.smithi105.stdout: "snap_mode": "selfmanaged", 2024-09-21T01:46:28.514 INFO:teuthology.orchestra.run.smithi105.stdout: "snap_seq": 0, 2024-09-21T01:46:28.514 INFO:teuthology.orchestra.run.smithi105.stdout: "snap_epoch": 0, 2024-09-21T01:46:28.514 INFO:teuthology.orchestra.run.smithi105.stdout: "pool_snaps": [], 2024-09-21T01:46:28.514 INFO:teuthology.orchestra.run.smithi105.stdout: "removed_snaps": "[]", 2024-09-21T01:46:28.514 INFO:teuthology.orchestra.run.smithi105.stdout: "quota_max_bytes": 0, 2024-09-21T01:46:28.514 INFO:teuthology.orchestra.run.smithi105.stdout: "quota_max_objects": 0, 2024-09-21T01:46:28.514 INFO:teuthology.orchestra.run.smithi105.stdout: "tiers": [], 2024-09-21T01:46:28.515 INFO:teuthology.orchestra.run.smithi105.stdout: "tier_of": -1, 2024-09-21T01:46:28.515 INFO:teuthology.orchestra.run.smithi105.stdout: "read_tier": -1, 2024-09-21T01:46:28.515 INFO:teuthology.orchestra.run.smithi105.stdout: "write_tier": -1, 2024-09-21T01:46:28.515 INFO:teuthology.orchestra.run.smithi105.stdout: "cache_mode": "none", 2024-09-21T01:46:28.515 INFO:teuthology.orchestra.run.smithi105.stdout: "target_max_bytes": 0, 2024-09-21T01:46:28.515 INFO:teuthology.orchestra.run.smithi105.stdout: "target_max_objects": 0, 2024-09-21T01:46:28.515 INFO:teuthology.orchestra.run.smithi105.stdout: "cache_target_dirty_ratio_micro": 400000, 2024-09-21T01:46:28.515 INFO:teuthology.orchestra.run.smithi105.stdout: "cache_target_dirty_high_ratio_micro": 600000, 2024-09-21T01:46:28.515 INFO:teuthology.orchestra.run.smithi105.stdout: "cache_target_full_ratio_micro": 800000, 2024-09-21T01:46:28.515 INFO:teuthology.orchestra.run.smithi105.stdout: "cache_min_flush_age": 0, 2024-09-21T01:46:28.515 INFO:teuthology.orchestra.run.smithi105.stdout: "cache_min_evict_age": 0, 2024-09-21T01:46:28.515 INFO:teuthology.orchestra.run.smithi105.stdout: "erasure_code_profile": "", 2024-09-21T01:46:28.515 INFO:teuthology.orchestra.run.smithi105.stdout: "hit_set_params": { 2024-09-21T01:46:28.515 INFO:teuthology.orchestra.run.smithi105.stdout: "type": "none" 2024-09-21T01:46:28.516 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.516 INFO:teuthology.orchestra.run.smithi105.stdout: "hit_set_period": 0, 2024-09-21T01:46:28.516 INFO:teuthology.orchestra.run.smithi105.stdout: "hit_set_count": 0, 2024-09-21T01:46:28.516 INFO:teuthology.orchestra.run.smithi105.stdout: "use_gmt_hitset": true, 2024-09-21T01:46:28.516 INFO:teuthology.orchestra.run.smithi105.stdout: "min_read_recency_for_promote": 0, 2024-09-21T01:46:28.516 INFO:teuthology.orchestra.run.smithi105.stdout: "min_write_recency_for_promote": 0, 2024-09-21T01:46:28.516 INFO:teuthology.orchestra.run.smithi105.stdout: "hit_set_grade_decay_rate": 0, 2024-09-21T01:46:28.516 INFO:teuthology.orchestra.run.smithi105.stdout: "hit_set_search_last_n": 0, 2024-09-21T01:46:28.516 INFO:teuthology.orchestra.run.smithi105.stdout: "grade_table": [], 2024-09-21T01:46:28.516 INFO:teuthology.orchestra.run.smithi105.stdout: "stripe_width": 0, 2024-09-21T01:46:28.516 INFO:teuthology.orchestra.run.smithi105.stdout: "expected_num_objects": 0, 2024-09-21T01:46:28.516 INFO:teuthology.orchestra.run.smithi105.stdout: "fast_read": false, 2024-09-21T01:46:28.516 INFO:teuthology.orchestra.run.smithi105.stdout: "options": { 2024-09-21T01:46:28.516 INFO:teuthology.orchestra.run.smithi105.stdout: "pg_num_max": 32, 2024-09-21T01:46:28.517 INFO:teuthology.orchestra.run.smithi105.stdout: "pg_num_min": 1 2024-09-21T01:46:28.517 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.517 INFO:teuthology.orchestra.run.smithi105.stdout: "application_metadata": { 2024-09-21T01:46:28.517 INFO:teuthology.orchestra.run.smithi105.stdout: "mgr": {} 2024-09-21T01:46:28.517 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.517 INFO:teuthology.orchestra.run.smithi105.stdout: "read_balance": { 2024-09-21T01:46:28.517 INFO:teuthology.orchestra.run.smithi105.stdout: "score_type": "Fair distribution", 2024-09-21T01:46:28.517 INFO:teuthology.orchestra.run.smithi105.stdout: "score_acting": 4, 2024-09-21T01:46:28.517 INFO:teuthology.orchestra.run.smithi105.stdout: "score_stable": 4, 2024-09-21T01:46:28.517 INFO:teuthology.orchestra.run.smithi105.stdout: "optimal_score": 0.5, 2024-09-21T01:46:28.517 INFO:teuthology.orchestra.run.smithi105.stdout: "raw_score_acting": 2, 2024-09-21T01:46:28.517 INFO:teuthology.orchestra.run.smithi105.stdout: "raw_score_stable": 2, 2024-09-21T01:46:28.517 INFO:teuthology.orchestra.run.smithi105.stdout: "primary_affinity_weighted": 1, 2024-09-21T01:46:28.517 INFO:teuthology.orchestra.run.smithi105.stdout: "average_primary_affinity": 1, 2024-09-21T01:46:28.518 INFO:teuthology.orchestra.run.smithi105.stdout: "average_primary_affinity_weighted": 1 2024-09-21T01:46:28.518 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.518 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.518 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.518 INFO:teuthology.orchestra.run.smithi105.stdout: "pool": 2, 2024-09-21T01:46:28.518 INFO:teuthology.orchestra.run.smithi105.stdout: "pool_name": "rbd", 2024-09-21T01:46:28.518 INFO:teuthology.orchestra.run.smithi105.stdout: "create_time": "2024-09-21T01:44:54.210449+0000", 2024-09-21T01:46:28.518 INFO:teuthology.orchestra.run.smithi105.stdout: "flags": 8193, 2024-09-21T01:46:28.518 INFO:teuthology.orchestra.run.smithi105.stdout: "flags_names": "hashpspool,selfmanaged_snaps", 2024-09-21T01:46:28.518 INFO:teuthology.orchestra.run.smithi105.stdout: "type": 1, 2024-09-21T01:46:28.518 INFO:teuthology.orchestra.run.smithi105.stdout: "size": 2, 2024-09-21T01:46:28.518 INFO:teuthology.orchestra.run.smithi105.stdout: "min_size": 1, 2024-09-21T01:46:28.518 INFO:teuthology.orchestra.run.smithi105.stdout: "crush_rule": 0, 2024-09-21T01:46:28.518 INFO:teuthology.orchestra.run.smithi105.stdout: "peering_crush_bucket_count": 0, 2024-09-21T01:46:28.519 INFO:teuthology.orchestra.run.smithi105.stdout: "peering_crush_bucket_target": 0, 2024-09-21T01:46:28.519 INFO:teuthology.orchestra.run.smithi105.stdout: "peering_crush_bucket_barrier": 0, 2024-09-21T01:46:28.519 INFO:teuthology.orchestra.run.smithi105.stdout: "peering_crush_bucket_mandatory_member": 2147483647, 2024-09-21T01:46:28.519 INFO:teuthology.orchestra.run.smithi105.stdout: "is_stretch_pool": false, 2024-09-21T01:46:28.519 INFO:teuthology.orchestra.run.smithi105.stdout: "object_hash": 2, 2024-09-21T01:46:28.519 INFO:teuthology.orchestra.run.smithi105.stdout: "pg_autoscale_mode": "off", 2024-09-21T01:46:28.519 INFO:teuthology.orchestra.run.smithi105.stdout: "pg_num": 8, 2024-09-21T01:46:28.519 INFO:teuthology.orchestra.run.smithi105.stdout: "pg_placement_num": 8, 2024-09-21T01:46:28.519 INFO:teuthology.orchestra.run.smithi105.stdout: "pg_placement_num_target": 8, 2024-09-21T01:46:28.519 INFO:teuthology.orchestra.run.smithi105.stdout: "pg_num_target": 8, 2024-09-21T01:46:28.519 INFO:teuthology.orchestra.run.smithi105.stdout: "pg_num_pending": 8, 2024-09-21T01:46:28.519 INFO:teuthology.orchestra.run.smithi105.stdout: "last_pg_merge_meta": { 2024-09-21T01:46:28.519 INFO:teuthology.orchestra.run.smithi105.stdout: "source_pgid": "0.0", 2024-09-21T01:46:28.519 INFO:teuthology.orchestra.run.smithi105.stdout: "ready_epoch": 0, 2024-09-21T01:46:28.520 INFO:teuthology.orchestra.run.smithi105.stdout: "last_epoch_started": 0, 2024-09-21T01:46:28.520 INFO:teuthology.orchestra.run.smithi105.stdout: "last_epoch_clean": 0, 2024-09-21T01:46:28.520 INFO:teuthology.orchestra.run.smithi105.stdout: "source_version": "0'0", 2024-09-21T01:46:28.520 INFO:teuthology.orchestra.run.smithi105.stdout: "target_version": "0'0" 2024-09-21T01:46:28.520 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.520 INFO:teuthology.orchestra.run.smithi105.stdout: "last_change": "17", 2024-09-21T01:46:28.520 INFO:teuthology.orchestra.run.smithi105.stdout: "last_force_op_resend": "0", 2024-09-21T01:46:28.520 INFO:teuthology.orchestra.run.smithi105.stdout: "last_force_op_resend_prenautilus": "0", 2024-09-21T01:46:28.520 INFO:teuthology.orchestra.run.smithi105.stdout: "last_force_op_resend_preluminous": "0", 2024-09-21T01:46:28.520 INFO:teuthology.orchestra.run.smithi105.stdout: "auid": 0, 2024-09-21T01:46:28.520 INFO:teuthology.orchestra.run.smithi105.stdout: "snap_mode": "selfmanaged", 2024-09-21T01:46:28.520 INFO:teuthology.orchestra.run.smithi105.stdout: "snap_seq": 3, 2024-09-21T01:46:28.520 INFO:teuthology.orchestra.run.smithi105.stdout: "snap_epoch": 17, 2024-09-21T01:46:28.520 INFO:teuthology.orchestra.run.smithi105.stdout: "pool_snaps": [], 2024-09-21T01:46:28.520 INFO:teuthology.orchestra.run.smithi105.stdout: "removed_snaps": "[]", 2024-09-21T01:46:28.521 INFO:teuthology.orchestra.run.smithi105.stdout: "quota_max_bytes": 0, 2024-09-21T01:46:28.521 INFO:teuthology.orchestra.run.smithi105.stdout: "quota_max_objects": 0, 2024-09-21T01:46:28.521 INFO:teuthology.orchestra.run.smithi105.stdout: "tiers": [], 2024-09-21T01:46:28.521 INFO:teuthology.orchestra.run.smithi105.stdout: "tier_of": -1, 2024-09-21T01:46:28.521 INFO:teuthology.orchestra.run.smithi105.stdout: "read_tier": -1, 2024-09-21T01:46:28.521 INFO:teuthology.orchestra.run.smithi105.stdout: "write_tier": -1, 2024-09-21T01:46:28.521 INFO:teuthology.orchestra.run.smithi105.stdout: "cache_mode": "none", 2024-09-21T01:46:28.521 INFO:teuthology.orchestra.run.smithi105.stdout: "target_max_bytes": 0, 2024-09-21T01:46:28.521 INFO:teuthology.orchestra.run.smithi105.stdout: "target_max_objects": 0, 2024-09-21T01:46:28.521 INFO:teuthology.orchestra.run.smithi105.stdout: "cache_target_dirty_ratio_micro": 400000, 2024-09-21T01:46:28.521 INFO:teuthology.orchestra.run.smithi105.stdout: "cache_target_dirty_high_ratio_micro": 600000, 2024-09-21T01:46:28.521 INFO:teuthology.orchestra.run.smithi105.stdout: "cache_target_full_ratio_micro": 800000, 2024-09-21T01:46:28.521 INFO:teuthology.orchestra.run.smithi105.stdout: "cache_min_flush_age": 0, 2024-09-21T01:46:28.521 INFO:teuthology.orchestra.run.smithi105.stdout: "cache_min_evict_age": 0, 2024-09-21T01:46:28.522 INFO:teuthology.orchestra.run.smithi105.stdout: "erasure_code_profile": "", 2024-09-21T01:46:28.522 INFO:teuthology.orchestra.run.smithi105.stdout: "hit_set_params": { 2024-09-21T01:46:28.522 INFO:teuthology.orchestra.run.smithi105.stdout: "type": "none" 2024-09-21T01:46:28.522 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.522 INFO:teuthology.orchestra.run.smithi105.stdout: "hit_set_period": 0, 2024-09-21T01:46:28.522 INFO:teuthology.orchestra.run.smithi105.stdout: "hit_set_count": 0, 2024-09-21T01:46:28.522 INFO:teuthology.orchestra.run.smithi105.stdout: "use_gmt_hitset": true, 2024-09-21T01:46:28.522 INFO:teuthology.orchestra.run.smithi105.stdout: "min_read_recency_for_promote": 0, 2024-09-21T01:46:28.522 INFO:teuthology.orchestra.run.smithi105.stdout: "min_write_recency_for_promote": 0, 2024-09-21T01:46:28.522 INFO:teuthology.orchestra.run.smithi105.stdout: "hit_set_grade_decay_rate": 0, 2024-09-21T01:46:28.522 INFO:teuthology.orchestra.run.smithi105.stdout: "hit_set_search_last_n": 0, 2024-09-21T01:46:28.522 INFO:teuthology.orchestra.run.smithi105.stdout: "grade_table": [], 2024-09-21T01:46:28.522 INFO:teuthology.orchestra.run.smithi105.stdout: "stripe_width": 0, 2024-09-21T01:46:28.522 INFO:teuthology.orchestra.run.smithi105.stdout: "expected_num_objects": 0, 2024-09-21T01:46:28.523 INFO:teuthology.orchestra.run.smithi105.stdout: "fast_read": false, 2024-09-21T01:46:28.523 INFO:teuthology.orchestra.run.smithi105.stdout: "options": {}, 2024-09-21T01:46:28.523 INFO:teuthology.orchestra.run.smithi105.stdout: "application_metadata": { 2024-09-21T01:46:28.523 INFO:teuthology.orchestra.run.smithi105.stdout: "rbd": {} 2024-09-21T01:46:28.523 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.523 INFO:teuthology.orchestra.run.smithi105.stdout: "read_balance": { 2024-09-21T01:46:28.523 INFO:teuthology.orchestra.run.smithi105.stdout: "score_type": "Fair distribution", 2024-09-21T01:46:28.523 INFO:teuthology.orchestra.run.smithi105.stdout: "score_acting": 2.5, 2024-09-21T01:46:28.523 INFO:teuthology.orchestra.run.smithi105.stdout: "score_stable": 2.5, 2024-09-21T01:46:28.523 INFO:teuthology.orchestra.run.smithi105.stdout: "optimal_score": 1, 2024-09-21T01:46:28.523 INFO:teuthology.orchestra.run.smithi105.stdout: "raw_score_acting": 2.5, 2024-09-21T01:46:28.523 INFO:teuthology.orchestra.run.smithi105.stdout: "raw_score_stable": 2.5, 2024-09-21T01:46:28.523 INFO:teuthology.orchestra.run.smithi105.stdout: "primary_affinity_weighted": 1, 2024-09-21T01:46:28.525 INFO:teuthology.orchestra.run.smithi105.stdout: "average_primary_affinity": 1, 2024-09-21T01:46:28.525 INFO:teuthology.orchestra.run.smithi105.stdout: "average_primary_affinity_weighted": 1 2024-09-21T01:46:28.525 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.525 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.525 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-21T01:46:28.526 INFO:teuthology.orchestra.run.smithi105.stdout: "osds": [ 2024-09-21T01:46:28.526 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.526 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": 0, 2024-09-21T01:46:28.526 INFO:teuthology.orchestra.run.smithi105.stdout: "uuid": "03218bdb-ec48-44c7-bbf5-31cb53a3cea7", 2024-09-21T01:46:28.526 INFO:teuthology.orchestra.run.smithi105.stdout: "up": 1, 2024-09-21T01:46:28.526 INFO:teuthology.orchestra.run.smithi105.stdout: "in": 1, 2024-09-21T01:46:28.526 INFO:teuthology.orchestra.run.smithi105.stdout: "weight": 1, 2024-09-21T01:46:28.526 INFO:teuthology.orchestra.run.smithi105.stdout: "primary_affinity": 1, 2024-09-21T01:46:28.526 INFO:teuthology.orchestra.run.smithi105.stdout: "last_clean_begin": 9, 2024-09-21T01:46:28.526 INFO:teuthology.orchestra.run.smithi105.stdout: "last_clean_end": 20, 2024-09-21T01:46:28.526 INFO:teuthology.orchestra.run.smithi105.stdout: "up_from": 26, 2024-09-21T01:46:28.527 INFO:teuthology.orchestra.run.smithi105.stdout: "up_thru": 26, 2024-09-21T01:46:28.527 INFO:teuthology.orchestra.run.smithi105.stdout: "down_at": 21, 2024-09-21T01:46:28.527 INFO:teuthology.orchestra.run.smithi105.stdout: "lost_at": 0, 2024-09-21T01:46:28.527 INFO:teuthology.orchestra.run.smithi105.stdout: "public_addrs": { 2024-09-21T01:46:28.527 INFO:teuthology.orchestra.run.smithi105.stdout: "addrvec": [ 2024-09-21T01:46:28.527 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.527 INFO:teuthology.orchestra.run.smithi105.stdout: "type": "v1", 2024-09-21T01:46:28.527 INFO:teuthology.orchestra.run.smithi105.stdout: "addr": "172.21.15.105:6800", 2024-09-21T01:46:28.527 INFO:teuthology.orchestra.run.smithi105.stdout: "nonce": 249246859 2024-09-21T01:46:28.527 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.527 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-21T01:46:28.527 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.527 INFO:teuthology.orchestra.run.smithi105.stdout: "cluster_addrs": { 2024-09-21T01:46:28.527 INFO:teuthology.orchestra.run.smithi105.stdout: "addrvec": [ 2024-09-21T01:46:28.528 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.528 INFO:teuthology.orchestra.run.smithi105.stdout: "type": "v1", 2024-09-21T01:46:28.528 INFO:teuthology.orchestra.run.smithi105.stdout: "addr": "172.21.15.105:6801", 2024-09-21T01:46:28.528 INFO:teuthology.orchestra.run.smithi105.stdout: "nonce": 249246859 2024-09-21T01:46:28.528 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.528 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-21T01:46:28.528 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.528 INFO:teuthology.orchestra.run.smithi105.stdout: "heartbeat_back_addrs": { 2024-09-21T01:46:28.528 INFO:teuthology.orchestra.run.smithi105.stdout: "addrvec": [ 2024-09-21T01:46:28.528 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.528 INFO:teuthology.orchestra.run.smithi105.stdout: "type": "v1", 2024-09-21T01:46:28.528 INFO:teuthology.orchestra.run.smithi105.stdout: "addr": "172.21.15.105:6803", 2024-09-21T01:46:28.528 INFO:teuthology.orchestra.run.smithi105.stdout: "nonce": 249246859 2024-09-21T01:46:28.529 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.529 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-21T01:46:28.529 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.529 INFO:teuthology.orchestra.run.smithi105.stdout: "heartbeat_front_addrs": { 2024-09-21T01:46:28.529 INFO:teuthology.orchestra.run.smithi105.stdout: "addrvec": [ 2024-09-21T01:46:28.529 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.529 INFO:teuthology.orchestra.run.smithi105.stdout: "type": "v1", 2024-09-21T01:46:28.529 INFO:teuthology.orchestra.run.smithi105.stdout: "addr": "172.21.15.105:6802", 2024-09-21T01:46:28.529 INFO:teuthology.orchestra.run.smithi105.stdout: "nonce": 249246859 2024-09-21T01:46:28.529 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.529 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-21T01:46:28.529 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.529 INFO:teuthology.orchestra.run.smithi105.stdout: "public_addr": "172.21.15.105:6800/249246859", 2024-09-21T01:46:28.529 INFO:teuthology.orchestra.run.smithi105.stdout: "cluster_addr": "172.21.15.105:6801/249246859", 2024-09-21T01:46:28.529 INFO:teuthology.orchestra.run.smithi105.stdout: "heartbeat_back_addr": "172.21.15.105:6803/249246859", 2024-09-21T01:46:28.530 INFO:teuthology.orchestra.run.smithi105.stdout: "heartbeat_front_addr": "172.21.15.105:6802/249246859", 2024-09-21T01:46:28.530 INFO:teuthology.orchestra.run.smithi105.stdout: "state": [ 2024-09-21T01:46:28.530 INFO:teuthology.orchestra.run.smithi105.stdout: "exists", 2024-09-21T01:46:28.530 INFO:teuthology.orchestra.run.smithi105.stdout: "up" 2024-09-21T01:46:28.530 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-21T01:46:28.530 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.530 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.530 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": 1, 2024-09-21T01:46:28.530 INFO:teuthology.orchestra.run.smithi105.stdout: "uuid": "5528ff44-922a-4e44-bb69-95fd9544d8dd", 2024-09-21T01:46:28.530 INFO:teuthology.orchestra.run.smithi105.stdout: "up": 1, 2024-09-21T01:46:28.530 INFO:teuthology.orchestra.run.smithi105.stdout: "in": 1, 2024-09-21T01:46:28.530 INFO:teuthology.orchestra.run.smithi105.stdout: "weight": 1, 2024-09-21T01:46:28.530 INFO:teuthology.orchestra.run.smithi105.stdout: "primary_affinity": 1, 2024-09-21T01:46:28.530 INFO:teuthology.orchestra.run.smithi105.stdout: "last_clean_begin": 9, 2024-09-21T01:46:28.531 INFO:teuthology.orchestra.run.smithi105.stdout: "last_clean_end": 20, 2024-09-21T01:46:28.531 INFO:teuthology.orchestra.run.smithi105.stdout: "up_from": 27, 2024-09-21T01:46:28.531 INFO:teuthology.orchestra.run.smithi105.stdout: "up_thru": 27, 2024-09-21T01:46:28.531 INFO:teuthology.orchestra.run.smithi105.stdout: "down_at": 22, 2024-09-21T01:46:28.531 INFO:teuthology.orchestra.run.smithi105.stdout: "lost_at": 0, 2024-09-21T01:46:28.531 INFO:teuthology.orchestra.run.smithi105.stdout: "public_addrs": { 2024-09-21T01:46:28.531 INFO:teuthology.orchestra.run.smithi105.stdout: "addrvec": [ 2024-09-21T01:46:28.531 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.531 INFO:teuthology.orchestra.run.smithi105.stdout: "type": "v1", 2024-09-21T01:46:28.531 INFO:teuthology.orchestra.run.smithi105.stdout: "addr": "172.21.15.105:6804", 2024-09-21T01:46:28.531 INFO:teuthology.orchestra.run.smithi105.stdout: "nonce": 1896987326 2024-09-21T01:46:28.531 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.531 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-21T01:46:28.531 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.532 INFO:teuthology.orchestra.run.smithi105.stdout: "cluster_addrs": { 2024-09-21T01:46:28.532 INFO:teuthology.orchestra.run.smithi105.stdout: "addrvec": [ 2024-09-21T01:46:28.532 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.532 INFO:teuthology.orchestra.run.smithi105.stdout: "type": "v1", 2024-09-21T01:46:28.532 INFO:teuthology.orchestra.run.smithi105.stdout: "addr": "172.21.15.105:6805", 2024-09-21T01:46:28.532 INFO:teuthology.orchestra.run.smithi105.stdout: "nonce": 1896987326 2024-09-21T01:46:28.532 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.532 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-21T01:46:28.532 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.532 INFO:teuthology.orchestra.run.smithi105.stdout: "heartbeat_back_addrs": { 2024-09-21T01:46:28.532 INFO:teuthology.orchestra.run.smithi105.stdout: "addrvec": [ 2024-09-21T01:46:28.532 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.532 INFO:teuthology.orchestra.run.smithi105.stdout: "type": "v1", 2024-09-21T01:46:28.532 INFO:teuthology.orchestra.run.smithi105.stdout: "addr": "172.21.15.105:6807", 2024-09-21T01:46:28.533 INFO:teuthology.orchestra.run.smithi105.stdout: "nonce": 1896987326 2024-09-21T01:46:28.533 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.533 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-21T01:46:28.533 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.533 INFO:teuthology.orchestra.run.smithi105.stdout: "heartbeat_front_addrs": { 2024-09-21T01:46:28.533 INFO:teuthology.orchestra.run.smithi105.stdout: "addrvec": [ 2024-09-21T01:46:28.533 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.533 INFO:teuthology.orchestra.run.smithi105.stdout: "type": "v1", 2024-09-21T01:46:28.533 INFO:teuthology.orchestra.run.smithi105.stdout: "addr": "172.21.15.105:6806", 2024-09-21T01:46:28.533 INFO:teuthology.orchestra.run.smithi105.stdout: "nonce": 1896987326 2024-09-21T01:46:28.533 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.533 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-21T01:46:28.533 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.533 INFO:teuthology.orchestra.run.smithi105.stdout: "public_addr": "172.21.15.105:6804/1896987326", 2024-09-21T01:46:28.533 INFO:teuthology.orchestra.run.smithi105.stdout: "cluster_addr": "172.21.15.105:6805/1896987326", 2024-09-21T01:46:28.534 INFO:teuthology.orchestra.run.smithi105.stdout: "heartbeat_back_addr": "172.21.15.105:6807/1896987326", 2024-09-21T01:46:28.534 INFO:teuthology.orchestra.run.smithi105.stdout: "heartbeat_front_addr": "172.21.15.105:6806/1896987326", 2024-09-21T01:46:28.534 INFO:teuthology.orchestra.run.smithi105.stdout: "state": [ 2024-09-21T01:46:28.534 INFO:teuthology.orchestra.run.smithi105.stdout: "exists", 2024-09-21T01:46:28.534 INFO:teuthology.orchestra.run.smithi105.stdout: "up" 2024-09-21T01:46:28.534 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-21T01:46:28.534 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.534 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.534 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": 2, 2024-09-21T01:46:28.534 INFO:teuthology.orchestra.run.smithi105.stdout: "uuid": "df56ff35-71e9-4d46-8abb-84265eb6e1ee", 2024-09-21T01:46:28.534 INFO:teuthology.orchestra.run.smithi105.stdout: "up": 1, 2024-09-21T01:46:28.534 INFO:teuthology.orchestra.run.smithi105.stdout: "in": 1, 2024-09-21T01:46:28.534 INFO:teuthology.orchestra.run.smithi105.stdout: "weight": 1, 2024-09-21T01:46:28.535 INFO:teuthology.orchestra.run.smithi105.stdout: "primary_affinity": 1, 2024-09-21T01:46:28.535 INFO:teuthology.orchestra.run.smithi105.stdout: "last_clean_begin": 9, 2024-09-21T01:46:28.535 INFO:teuthology.orchestra.run.smithi105.stdout: "last_clean_end": 22, 2024-09-21T01:46:28.535 INFO:teuthology.orchestra.run.smithi105.stdout: "up_from": 29, 2024-09-21T01:46:28.535 INFO:teuthology.orchestra.run.smithi105.stdout: "up_thru": 14, 2024-09-21T01:46:28.535 INFO:teuthology.orchestra.run.smithi105.stdout: "down_at": 23, 2024-09-21T01:46:28.535 INFO:teuthology.orchestra.run.smithi105.stdout: "lost_at": 0, 2024-09-21T01:46:28.535 INFO:teuthology.orchestra.run.smithi105.stdout: "public_addrs": { 2024-09-21T01:46:28.535 INFO:teuthology.orchestra.run.smithi105.stdout: "addrvec": [ 2024-09-21T01:46:28.535 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.535 INFO:teuthology.orchestra.run.smithi105.stdout: "type": "v1", 2024-09-21T01:46:28.535 INFO:teuthology.orchestra.run.smithi105.stdout: "addr": "172.21.15.105:6808", 2024-09-21T01:46:28.535 INFO:teuthology.orchestra.run.smithi105.stdout: "nonce": 2933198036 2024-09-21T01:46:28.535 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.536 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-21T01:46:28.536 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.536 INFO:teuthology.orchestra.run.smithi105.stdout: "cluster_addrs": { 2024-09-21T01:46:28.536 INFO:teuthology.orchestra.run.smithi105.stdout: "addrvec": [ 2024-09-21T01:46:28.536 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.536 INFO:teuthology.orchestra.run.smithi105.stdout: "type": "v1", 2024-09-21T01:46:28.536 INFO:teuthology.orchestra.run.smithi105.stdout: "addr": "172.21.15.105:6809", 2024-09-21T01:46:28.536 INFO:teuthology.orchestra.run.smithi105.stdout: "nonce": 2933198036 2024-09-21T01:46:28.536 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.536 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-21T01:46:28.536 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.536 INFO:teuthology.orchestra.run.smithi105.stdout: "heartbeat_back_addrs": { 2024-09-21T01:46:28.536 INFO:teuthology.orchestra.run.smithi105.stdout: "addrvec": [ 2024-09-21T01:46:28.536 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.537 INFO:teuthology.orchestra.run.smithi105.stdout: "type": "v1", 2024-09-21T01:46:28.537 INFO:teuthology.orchestra.run.smithi105.stdout: "addr": "172.21.15.105:6811", 2024-09-21T01:46:28.537 INFO:teuthology.orchestra.run.smithi105.stdout: "nonce": 2933198036 2024-09-21T01:46:28.537 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.537 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-21T01:46:28.537 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.537 INFO:teuthology.orchestra.run.smithi105.stdout: "heartbeat_front_addrs": { 2024-09-21T01:46:28.537 INFO:teuthology.orchestra.run.smithi105.stdout: "addrvec": [ 2024-09-21T01:46:28.537 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.537 INFO:teuthology.orchestra.run.smithi105.stdout: "type": "v1", 2024-09-21T01:46:28.537 INFO:teuthology.orchestra.run.smithi105.stdout: "addr": "172.21.15.105:6810", 2024-09-21T01:46:28.537 INFO:teuthology.orchestra.run.smithi105.stdout: "nonce": 2933198036 2024-09-21T01:46:28.537 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.537 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-21T01:46:28.537 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.538 INFO:teuthology.orchestra.run.smithi105.stdout: "public_addr": "172.21.15.105:6808/2933198036", 2024-09-21T01:46:28.538 INFO:teuthology.orchestra.run.smithi105.stdout: "cluster_addr": "172.21.15.105:6809/2933198036", 2024-09-21T01:46:28.538 INFO:teuthology.orchestra.run.smithi105.stdout: "heartbeat_back_addr": "172.21.15.105:6811/2933198036", 2024-09-21T01:46:28.538 INFO:teuthology.orchestra.run.smithi105.stdout: "heartbeat_front_addr": "172.21.15.105:6810/2933198036", 2024-09-21T01:46:28.538 INFO:teuthology.orchestra.run.smithi105.stdout: "state": [ 2024-09-21T01:46:28.538 INFO:teuthology.orchestra.run.smithi105.stdout: "exists", 2024-09-21T01:46:28.538 INFO:teuthology.orchestra.run.smithi105.stdout: "up" 2024-09-21T01:46:28.538 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-21T01:46:28.538 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.538 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.538 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": 3, 2024-09-21T01:46:28.538 INFO:teuthology.orchestra.run.smithi105.stdout: "uuid": "012d9668-f1d3-4268-b0eb-15fa845ef17c", 2024-09-21T01:46:28.538 INFO:teuthology.orchestra.run.smithi105.stdout: "up": 1, 2024-09-21T01:46:28.538 INFO:teuthology.orchestra.run.smithi105.stdout: "in": 1, 2024-09-21T01:46:28.539 INFO:teuthology.orchestra.run.smithi105.stdout: "weight": 1, 2024-09-21T01:46:28.539 INFO:teuthology.orchestra.run.smithi105.stdout: "primary_affinity": 1, 2024-09-21T01:46:28.539 INFO:teuthology.orchestra.run.smithi105.stdout: "last_clean_begin": 9, 2024-09-21T01:46:28.539 INFO:teuthology.orchestra.run.smithi105.stdout: "last_clean_end": 23, 2024-09-21T01:46:28.539 INFO:teuthology.orchestra.run.smithi105.stdout: "up_from": 29, 2024-09-21T01:46:28.539 INFO:teuthology.orchestra.run.smithi105.stdout: "up_thru": 23, 2024-09-21T01:46:28.539 INFO:teuthology.orchestra.run.smithi105.stdout: "down_at": 24, 2024-09-21T01:46:28.539 INFO:teuthology.orchestra.run.smithi105.stdout: "lost_at": 0, 2024-09-21T01:46:28.539 INFO:teuthology.orchestra.run.smithi105.stdout: "public_addrs": { 2024-09-21T01:46:28.539 INFO:teuthology.orchestra.run.smithi105.stdout: "addrvec": [ 2024-09-21T01:46:28.539 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.539 INFO:teuthology.orchestra.run.smithi105.stdout: "type": "v1", 2024-09-21T01:46:28.539 INFO:teuthology.orchestra.run.smithi105.stdout: "addr": "172.21.15.105:6812", 2024-09-21T01:46:28.540 INFO:teuthology.orchestra.run.smithi105.stdout: "nonce": 568702146 2024-09-21T01:46:28.540 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.540 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-21T01:46:28.540 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.540 INFO:teuthology.orchestra.run.smithi105.stdout: "cluster_addrs": { 2024-09-21T01:46:28.540 INFO:teuthology.orchestra.run.smithi105.stdout: "addrvec": [ 2024-09-21T01:46:28.540 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.540 INFO:teuthology.orchestra.run.smithi105.stdout: "type": "v1", 2024-09-21T01:46:28.540 INFO:teuthology.orchestra.run.smithi105.stdout: "addr": "172.21.15.105:6813", 2024-09-21T01:46:28.540 INFO:teuthology.orchestra.run.smithi105.stdout: "nonce": 568702146 2024-09-21T01:46:28.540 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.540 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-21T01:46:28.540 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.540 INFO:teuthology.orchestra.run.smithi105.stdout: "heartbeat_back_addrs": { 2024-09-21T01:46:28.541 INFO:teuthology.orchestra.run.smithi105.stdout: "addrvec": [ 2024-09-21T01:46:28.541 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.541 INFO:teuthology.orchestra.run.smithi105.stdout: "type": "v1", 2024-09-21T01:46:28.541 INFO:teuthology.orchestra.run.smithi105.stdout: "addr": "172.21.15.105:6815", 2024-09-21T01:46:28.541 INFO:teuthology.orchestra.run.smithi105.stdout: "nonce": 568702146 2024-09-21T01:46:28.541 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.541 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-21T01:46:28.541 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.541 INFO:teuthology.orchestra.run.smithi105.stdout: "heartbeat_front_addrs": { 2024-09-21T01:46:28.541 INFO:teuthology.orchestra.run.smithi105.stdout: "addrvec": [ 2024-09-21T01:46:28.541 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.541 INFO:teuthology.orchestra.run.smithi105.stdout: "type": "v1", 2024-09-21T01:46:28.541 INFO:teuthology.orchestra.run.smithi105.stdout: "addr": "172.21.15.105:6814", 2024-09-21T01:46:28.541 INFO:teuthology.orchestra.run.smithi105.stdout: "nonce": 568702146 2024-09-21T01:46:28.541 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.542 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-21T01:46:28.542 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.542 INFO:teuthology.orchestra.run.smithi105.stdout: "public_addr": "172.21.15.105:6812/568702146", 2024-09-21T01:46:28.542 INFO:teuthology.orchestra.run.smithi105.stdout: "cluster_addr": "172.21.15.105:6813/568702146", 2024-09-21T01:46:28.542 INFO:teuthology.orchestra.run.smithi105.stdout: "heartbeat_back_addr": "172.21.15.105:6815/568702146", 2024-09-21T01:46:28.542 INFO:teuthology.orchestra.run.smithi105.stdout: "heartbeat_front_addr": "172.21.15.105:6814/568702146", 2024-09-21T01:46:28.542 INFO:teuthology.orchestra.run.smithi105.stdout: "state": [ 2024-09-21T01:46:28.542 INFO:teuthology.orchestra.run.smithi105.stdout: "exists", 2024-09-21T01:46:28.542 INFO:teuthology.orchestra.run.smithi105.stdout: "up" 2024-09-21T01:46:28.542 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-21T01:46:28.542 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.542 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-21T01:46:28.542 INFO:teuthology.orchestra.run.smithi105.stdout: "osd_xinfo": [ 2024-09-21T01:46:28.542 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.543 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": 0, 2024-09-21T01:46:28.543 INFO:teuthology.orchestra.run.smithi105.stdout: "down_stamp": "0.000000", 2024-09-21T01:46:28.543 INFO:teuthology.orchestra.run.smithi105.stdout: "laggy_probability": 0, 2024-09-21T01:46:28.543 INFO:teuthology.orchestra.run.smithi105.stdout: "laggy_interval": 0, 2024-09-21T01:46:28.543 INFO:teuthology.orchestra.run.smithi105.stdout: "features": 4540719139924082687, 2024-09-21T01:46:28.543 INFO:teuthology.orchestra.run.smithi105.stdout: "old_weight": 0, 2024-09-21T01:46:28.543 INFO:teuthology.orchestra.run.smithi105.stdout: "last_purged_snaps_scrub": "2024-09-21T01:44:44.554795+0000", 2024-09-21T01:46:28.543 INFO:teuthology.orchestra.run.smithi105.stdout: "dead_epoch": 21 2024-09-21T01:46:28.543 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.543 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.543 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": 1, 2024-09-21T01:46:28.543 INFO:teuthology.orchestra.run.smithi105.stdout: "down_stamp": "0.000000", 2024-09-21T01:46:28.543 INFO:teuthology.orchestra.run.smithi105.stdout: "laggy_probability": 0, 2024-09-21T01:46:28.543 INFO:teuthology.orchestra.run.smithi105.stdout: "laggy_interval": 0, 2024-09-21T01:46:28.544 INFO:teuthology.orchestra.run.smithi105.stdout: "features": 4540719139924082687, 2024-09-21T01:46:28.544 INFO:teuthology.orchestra.run.smithi105.stdout: "old_weight": 0, 2024-09-21T01:46:28.544 INFO:teuthology.orchestra.run.smithi105.stdout: "last_purged_snaps_scrub": "2024-09-21T01:44:44.869400+0000", 2024-09-21T01:46:28.544 INFO:teuthology.orchestra.run.smithi105.stdout: "dead_epoch": 22 2024-09-21T01:46:28.544 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.544 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.544 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": 2, 2024-09-21T01:46:28.544 INFO:teuthology.orchestra.run.smithi105.stdout: "down_stamp": "0.000000", 2024-09-21T01:46:28.544 INFO:teuthology.orchestra.run.smithi105.stdout: "laggy_probability": 0, 2024-09-21T01:46:28.544 INFO:teuthology.orchestra.run.smithi105.stdout: "laggy_interval": 0, 2024-09-21T01:46:28.544 INFO:teuthology.orchestra.run.smithi105.stdout: "features": 4540719139924082687, 2024-09-21T01:46:28.544 INFO:teuthology.orchestra.run.smithi105.stdout: "old_weight": 0, 2024-09-21T01:46:28.544 INFO:teuthology.orchestra.run.smithi105.stdout: "last_purged_snaps_scrub": "2024-09-21T01:44:44.601687+0000", 2024-09-21T01:46:28.544 INFO:teuthology.orchestra.run.smithi105.stdout: "dead_epoch": 23 2024-09-21T01:46:28.544 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.545 INFO:teuthology.orchestra.run.smithi105.stdout: { 2024-09-21T01:46:28.545 INFO:teuthology.orchestra.run.smithi105.stdout: "osd": 3, 2024-09-21T01:46:28.545 INFO:teuthology.orchestra.run.smithi105.stdout: "down_stamp": "2024-09-21T01:46:21.001569+0000", 2024-09-21T01:46:28.545 INFO:teuthology.orchestra.run.smithi105.stdout: "laggy_probability": 0, 2024-09-21T01:46:28.545 INFO:teuthology.orchestra.run.smithi105.stdout: "laggy_interval": 0, 2024-09-21T01:46:28.545 INFO:teuthology.orchestra.run.smithi105.stdout: "features": 4540719139924082687, 2024-09-21T01:46:28.545 INFO:teuthology.orchestra.run.smithi105.stdout: "old_weight": 0, 2024-09-21T01:46:28.545 INFO:teuthology.orchestra.run.smithi105.stdout: "last_purged_snaps_scrub": "2024-09-21T01:44:44.822680+0000", 2024-09-21T01:46:28.545 INFO:teuthology.orchestra.run.smithi105.stdout: "dead_epoch": 0 2024-09-21T01:46:28.545 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.545 INFO:teuthology.orchestra.run.smithi105.stdout: ], 2024-09-21T01:46:28.545 INFO:teuthology.orchestra.run.smithi105.stdout: "pg_upmap": [], 2024-09-21T01:46:28.545 INFO:teuthology.orchestra.run.smithi105.stdout: "pg_upmap_items": [], 2024-09-21T01:46:28.545 INFO:teuthology.orchestra.run.smithi105.stdout: "pg_upmap_primaries": [], 2024-09-21T01:46:28.545 INFO:teuthology.orchestra.run.smithi105.stdout: "pg_temp": [], 2024-09-21T01:46:28.545 INFO:teuthology.orchestra.run.smithi105.stdout: "primary_temp": [], 2024-09-21T01:46:28.546 INFO:teuthology.orchestra.run.smithi105.stdout: "blocklist": { 2024-09-21T01:46:28.546 INFO:teuthology.orchestra.run.smithi105.stdout: "172.21.15.105:0/3561680891": "2024-09-22T01:46:13.485080+0000", 2024-09-21T01:46:28.546 INFO:teuthology.orchestra.run.smithi105.stdout: "172.21.15.105:0/2570087590": "2024-09-22T01:46:13.485080+0000", 2024-09-21T01:46:28.546 INFO:teuthology.orchestra.run.smithi105.stdout: "172.21.15.105:0/3837276043": "2024-09-22T01:46:13.485080+0000", 2024-09-21T01:46:28.546 INFO:teuthology.orchestra.run.smithi105.stdout: "172.21.15.105:0/2810478014": "2024-09-22T01:46:13.485080+0000", 2024-09-21T01:46:28.546 INFO:teuthology.orchestra.run.smithi105.stdout: "172.21.15.105:6816/3933313069": "2024-09-22T01:46:13.485080+0000" 2024-09-21T01:46:28.546 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.546 INFO:teuthology.orchestra.run.smithi105.stdout: "range_blocklist": {}, 2024-09-21T01:46:28.546 INFO:teuthology.orchestra.run.smithi105.stdout: "erasure_code_profiles": { 2024-09-21T01:46:28.546 INFO:teuthology.orchestra.run.smithi105.stdout: "default": { 2024-09-21T01:46:28.546 INFO:teuthology.orchestra.run.smithi105.stdout: "crush-failure-domain": "osd", 2024-09-21T01:46:28.546 INFO:teuthology.orchestra.run.smithi105.stdout: "k": "2", 2024-09-21T01:46:28.546 INFO:teuthology.orchestra.run.smithi105.stdout: "m": "1", 2024-09-21T01:46:28.546 INFO:teuthology.orchestra.run.smithi105.stdout: "plugin": "jerasure", 2024-09-21T01:46:28.546 INFO:teuthology.orchestra.run.smithi105.stdout: "technique": "reed_sol_van" 2024-09-21T01:46:28.547 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.547 INFO:teuthology.orchestra.run.smithi105.stdout: }, 2024-09-21T01:46:28.547 INFO:teuthology.orchestra.run.smithi105.stdout: "removed_snaps_queue": [], 2024-09-21T01:46:28.547 INFO:teuthology.orchestra.run.smithi105.stdout: "new_removed_snaps": [], 2024-09-21T01:46:28.547 INFO:teuthology.orchestra.run.smithi105.stdout: "new_purged_snaps": [], 2024-09-21T01:46:28.547 INFO:teuthology.orchestra.run.smithi105.stdout: "crush_node_flags": {}, 2024-09-21T01:46:28.547 INFO:teuthology.orchestra.run.smithi105.stdout: "device_class_flags": {}, 2024-09-21T01:46:28.547 INFO:teuthology.orchestra.run.smithi105.stdout: "stretch_mode": { 2024-09-21T01:46:28.547 INFO:teuthology.orchestra.run.smithi105.stdout: "stretch_mode_enabled": false, 2024-09-21T01:46:28.547 INFO:teuthology.orchestra.run.smithi105.stdout: "stretch_bucket_count": 0, 2024-09-21T01:46:28.547 INFO:teuthology.orchestra.run.smithi105.stdout: "degraded_stretch_mode": 0, 2024-09-21T01:46:28.547 INFO:teuthology.orchestra.run.smithi105.stdout: "recovering_stretch_mode": 0, 2024-09-21T01:46:28.547 INFO:teuthology.orchestra.run.smithi105.stdout: "stretch_mode_bucket": 0 2024-09-21T01:46:28.547 INFO:teuthology.orchestra.run.smithi105.stdout: } 2024-09-21T01:46:28.547 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-21T01:46:28.549 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.508+0000 7f0c5cd07640 1 -- 172.21.15.105:0/3246887750 >> v1:172.21.15.105:6816/2838005468 conn(0x7f0c380514e0 legacy=0x7f0c380539f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:28.549 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.508+0000 7f0c5cd07640 1 -- 172.21.15.105:0/3246887750 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0c58096550 msgr2=0x7f0c5812e9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:28.549 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.508+0000 7f0c5cd07640 1 --2- 172.21.15.105:0/3246887750 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0c58096550 0x7f0c5812e9a0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f0c48009d10 tx=0x7f0c4800b690 comp rx=0 tx=0).stop 2024-09-21T01:46:28.549 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.508+0000 7f0c5cd07640 1 -- 172.21.15.105:0/3246887750 shutdown_connections 2024-09-21T01:46:28.549 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.508+0000 7f0c5cd07640 1 --2- 172.21.15.105:0/3246887750 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0c5812dd80 0x7f0c5812f420 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.549 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.508+0000 7f0c5cd07640 1 --2- 172.21.15.105:0/3246887750 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c5812d430 0x7f0c5812eee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.549 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.508+0000 7f0c5cd07640 1 --2- 172.21.15.105:0/3246887750 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0c58096550 0x7f0c5812e9a0 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.549 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.508+0000 7f0c5cd07640 1 -- 172.21.15.105:0/3246887750 >> 172.21.15.105:0/3246887750 conn(0x7f0c5809e1a0 msgr2=0x7f0c58129ec0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:28.549 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.508+0000 7f0c5cd07640 1 -- 172.21.15.105:0/3246887750 shutdown_connections 2024-09-21T01:46:28.549 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.508+0000 7f0c5cd07640 1 -- 172.21.15.105:0/3246887750 wait complete. 2024-09-21T01:46:28.550 DEBUG:teuthology.orchestra.run.smithi105:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph fs dump' 2024-09-21T01:46:28.664 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.663+0000 7f24cc407640 1 -- 172.21.15.105:0/1263512991 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f24c4135120 msgr2=0x7f24c4137540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:28.664 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.663+0000 7f24cc407640 1 --2- 172.21.15.105:0/1263512991 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f24c4135120 0x7f24c4137540 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f24bc005fd0 tx=0x7f24bc02aa10 comp rx=0 tx=0).stop 2024-09-21T01:46:28.664 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.663+0000 7f24cc407640 1 -- 172.21.15.105:0/1263512991 shutdown_connections 2024-09-21T01:46:28.664 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.663+0000 7f24cc407640 1 --2- 172.21.15.105:0/1263512991 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f24c4135120 0x7f24c4137540 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.664 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.663+0000 7f24cc407640 1 --2- 172.21.15.105:0/1263512991 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f24c4130100 0x7f24c4134be0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.664 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.663+0000 7f24cc407640 1 -- 172.21.15.105:0/1263512991 >> 172.21.15.105:0/1263512991 conn(0x7f24c409ed50 msgr2=0x7f24c409f160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:28.664 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.663+0000 7f24cc407640 1 -- 172.21.15.105:0/1263512991 shutdown_connections 2024-09-21T01:46:28.664 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.663+0000 7f24cc407640 1 -- 172.21.15.105:0/1263512991 wait complete. 2024-09-21T01:46:28.666 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.664+0000 7f24cc407640 1 Processor -- start 2024-09-21T01:46:28.667 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.664+0000 7f24cc407640 1 -- start start 2024-09-21T01:46:28.667 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.665+0000 7f24cc407640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f24c412f5e0 0x7f24c409b430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:28.667 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.665+0000 7f24cc407640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f24c4130100 0x7f24c409b970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:28.667 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.665+0000 7f24cc407640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f24c4135120 0x7f24c409beb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:28.667 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.665+0000 7f24cc407640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f24c4138690 con 0x7f24c4135120 2024-09-21T01:46:28.667 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.665+0000 7f24cc407640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f24c4138510 con 0x7f24c4130100 2024-09-21T01:46:28.667 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.665+0000 7f24cc407640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f24c4138810 con 0x7f24c412f5e0 2024-09-21T01:46:28.667 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.665+0000 7f24ca1a3640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f24c412f5e0 0x7f24c409b430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:28.667 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.665+0000 7f24c99a2640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f24c4130100 0x7f24c409b970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:28.667 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.665+0000 7f24ca1a3640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f24c412f5e0 0x7f24c409b430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:38778/0 (socket says 172.21.15.105:38778) 2024-09-21T01:46:28.667 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.665+0000 7f24ca1a3640 1 -- 172.21.15.105:0/276022918 learned_addr learned my addr 172.21.15.105:0/276022918 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:28.667 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.665+0000 7f24ca9a4640 1 --2- 172.21.15.105:0/276022918 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f24c4135120 0x7f24c409beb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:28.668 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.665+0000 7f24ca1a3640 1 -- 172.21.15.105:0/276022918 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f24c4130100 msgr2=0x7f24c409b970 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:28.668 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.665+0000 7f24ca1a3640 1 --2- 172.21.15.105:0/276022918 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f24c4130100 0x7f24c409b970 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.668 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.665+0000 7f24ca1a3640 1 -- 172.21.15.105:0/276022918 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f24c4135120 msgr2=0x7f24c409beb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:28.668 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.665+0000 7f24ca1a3640 1 --2- 172.21.15.105:0/276022918 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f24c4135120 0x7f24c409beb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.668 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.665+0000 7f24ca1a3640 1 -- 172.21.15.105:0/276022918 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f24c4131840 con 0x7f24c412f5e0 2024-09-21T01:46:28.668 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.665+0000 7f24ca9a4640 1 --2- 172.21.15.105:0/276022918 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f24c4135120 0x7f24c409beb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:28.668 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.665+0000 7f24ca1a3640 1 --2- 172.21.15.105:0/276022918 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f24c412f5e0 0x7f24c409b430 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f24c000c430 tx=0x7f24c000c8f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:28.668 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.666+0000 7f24baffd640 1 -- 172.21.15.105:0/276022918 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f24c0017070 con 0x7f24c412f5e0 2024-09-21T01:46:28.668 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.666+0000 7f24baffd640 1 -- 172.21.15.105:0/276022918 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f24c00053c0 con 0x7f24c412f5e0 2024-09-21T01:46:28.668 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.666+0000 7f24cc407640 1 -- 172.21.15.105:0/276022918 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f24c412b330 con 0x7f24c412f5e0 2024-09-21T01:46:28.668 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.666+0000 7f24baffd640 1 -- 172.21.15.105:0/276022918 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f24c0003850 con 0x7f24c412f5e0 2024-09-21T01:46:28.668 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.666+0000 7f24cc407640 1 -- 172.21.15.105:0/276022918 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f24c412b8a0 con 0x7f24c412f5e0 2024-09-21T01:46:28.669 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.667+0000 7f24cc407640 1 -- 172.21.15.105:0/276022918 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f24c412c4a0 con 0x7f24c412f5e0 2024-09-21T01:46:28.669 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.667+0000 7f24baffd640 1 -- 172.21.15.105:0/276022918 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f24c001a070 con 0x7f24c412f5e0 2024-09-21T01:46:28.669 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.667+0000 7f24baffd640 1 -- 172.21.15.105:0/276022918 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(30..30 src has 1..30) ==== 3027+0+0 (secure 0 0 0) 0x7f24c00683a0 con 0x7f24c412f5e0 2024-09-21T01:46:28.671 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.671+0000 7f24baffd640 1 -- 172.21.15.105:0/276022918 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f24c0067450 con 0x7f24c412f5e0 2024-09-21T01:46:28.843 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.842+0000 7f24cc407640 1 -- 172.21.15.105:0/276022918 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "fs dump"} v 0) -- 0x7f24c412f9f0 con 0x7f24c412f5e0 2024-09-21T01:46:28.843 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.843+0000 7f24baffd640 1 -- 172.21.15.105:0/276022918 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "fs dump"}]=0 dumped fsmap epoch 1 v1) ==== 75+0+396 (secure 0 0 0) 0x7f24c0066c10 con 0x7f24c412f5e0 2024-09-21T01:46:28.844 INFO:teuthology.orchestra.run.smithi105.stdout:e1 2024-09-21T01:46:28.844 INFO:teuthology.orchestra.run.smithi105.stdout:btime 1970-01-01T00:00:00:000000+0000 2024-09-21T01:46:28.844 INFO:teuthology.orchestra.run.smithi105.stdout:enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T01:46:28.844 INFO:teuthology.orchestra.run.smithi105.stdout:default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T01:46:28.844 INFO:teuthology.orchestra.run.smithi105.stdout:legacy client fscid: -1 2024-09-21T01:46:28.844 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:28.844 INFO:teuthology.orchestra.run.smithi105.stdout:No filesystems configured 2024-09-21T01:46:28.844 INFO:teuthology.orchestra.run.smithi105.stderr:dumped fsmap epoch 1 2024-09-21T01:46:28.845 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.845+0000 7f24cc407640 1 -- 172.21.15.105:0/276022918 >> v1:172.21.15.105:6816/2838005468 conn(0x7f24a80514e0 legacy=0x7f24a80539f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:28.845 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.845+0000 7f24cc407640 1 -- 172.21.15.105:0/276022918 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f24c412f5e0 msgr2=0x7f24c409b430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:28.845 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.845+0000 7f24cc407640 1 --2- 172.21.15.105:0/276022918 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f24c412f5e0 0x7f24c409b430 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f24c000c430 tx=0x7f24c000c8f0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.845 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.845+0000 7f24cc407640 1 -- 172.21.15.105:0/276022918 shutdown_connections 2024-09-21T01:46:28.846 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.845+0000 7f24cc407640 1 --2- 172.21.15.105:0/276022918 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f24c4135120 0x7f24c409beb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.846 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.845+0000 7f24cc407640 1 --2- 172.21.15.105:0/276022918 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f24c4130100 0x7f24c409b970 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.846 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.845+0000 7f24cc407640 1 --2- 172.21.15.105:0/276022918 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f24c412f5e0 0x7f24c409b430 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.846 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.845+0000 7f24cc407640 1 -- 172.21.15.105:0/276022918 >> 172.21.15.105:0/276022918 conn(0x7f24c409ed50 msgr2=0x7f24c4074220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:28.846 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.845+0000 7f24cc407640 1 -- 172.21.15.105:0/276022918 shutdown_connections 2024-09-21T01:46:28.846 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.845+0000 7f24cc407640 1 -- 172.21.15.105:0/276022918 wait complete. 2024-09-21T01:46:28.856 DEBUG:teuthology.orchestra.run.smithi105:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph osd require-osd-release squid' 2024-09-21T01:46:28.964 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.964+0000 7ff074773640 1 -- 172.21.15.105:0/533842833 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff06c12cf20 msgr2=0x7ff06c131710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:28.965 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.964+0000 7ff074773640 1 --2- 172.21.15.105:0/533842833 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff06c12cf20 0x7ff06c131710 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7ff05c009830 tx=0x7ff05c02a480 comp rx=0 tx=0).stop 2024-09-21T01:46:28.965 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.964+0000 7ff074773640 1 -- 172.21.15.105:0/533842833 shutdown_connections 2024-09-21T01:46:28.965 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.964+0000 7ff074773640 1 --2- 172.21.15.105:0/533842833 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff06c131c50 0x7ff06c134070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.965 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.964+0000 7ff074773640 1 --2- 172.21.15.105:0/533842833 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff06c12cf20 0x7ff06c131710 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.965 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.964+0000 7ff074773640 1 -- 172.21.15.105:0/533842833 >> 172.21.15.105:0/533842833 conn(0x7ff06c09e270 msgr2=0x7ff06c09a520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:28.965 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.964+0000 7ff074773640 1 -- 172.21.15.105:0/533842833 shutdown_connections 2024-09-21T01:46:28.965 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.964+0000 7ff074773640 1 -- 172.21.15.105:0/533842833 wait complete. 2024-09-21T01:46:28.967 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.965+0000 7ff074773640 1 Processor -- start 2024-09-21T01:46:28.967 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.965+0000 7ff074773640 1 -- start start 2024-09-21T01:46:28.967 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.965+0000 7ff074773640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff06c072c80 0x7ff06c097620 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:28.967 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.965+0000 7ff074773640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff06c12cf20 0x7ff06c097b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:28.967 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.965+0000 7ff074773640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff06c131c50 0x7ff06c124ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:28.967 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.965+0000 7ff074773640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7ff06c1353a0 con 0x7ff06c12cf20 2024-09-21T01:46:28.968 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.965+0000 7ff074773640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7ff06c135220 con 0x7ff06c131c50 2024-09-21T01:46:28.968 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.965+0000 7ff074773640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7ff06c135520 con 0x7ff06c072c80 2024-09-21T01:46:28.968 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.966+0000 7ff071d0e640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff06c12cf20 0x7ff06c097b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:28.968 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.966+0000 7ff07250f640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff06c072c80 0x7ff06c097620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:28.968 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.966+0000 7ff072d10640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff06c131c50 0x7ff06c124ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:28.968 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.966+0000 7ff071d0e640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff06c12cf20 0x7ff06c097b60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:34898/0 (socket says 172.21.15.105:34898) 2024-09-21T01:46:28.968 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.966+0000 7ff07250f640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff06c072c80 0x7ff06c097620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:38788/0 (socket says 172.21.15.105:38788) 2024-09-21T01:46:28.968 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.966+0000 7ff071d0e640 1 -- 172.21.15.105:0/1065999043 learned_addr learned my addr 172.21.15.105:0/1065999043 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:28.968 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.966+0000 7ff071d0e640 1 -- 172.21.15.105:0/1065999043 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff06c072c80 msgr2=0x7ff06c097620 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:28.968 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.966+0000 7ff071d0e640 1 --2- 172.21.15.105:0/1065999043 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff06c072c80 0x7ff06c097620 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.968 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.966+0000 7ff071d0e640 1 -- 172.21.15.105:0/1065999043 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff06c131c50 msgr2=0x7ff06c124ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:28.968 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.966+0000 7ff071d0e640 1 --2- 172.21.15.105:0/1065999043 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff06c131c50 0x7ff06c124ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:28.968 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.966+0000 7ff071d0e640 1 -- 172.21.15.105:0/1065999043 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff06c1269e0 con 0x7ff06c12cf20 2024-09-21T01:46:28.969 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.966+0000 7ff072d10640 1 --2- 172.21.15.105:0/1065999043 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff06c131c50 0x7ff06c124ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:28.969 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.966+0000 7ff07250f640 1 --2- 172.21.15.105:0/1065999043 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff06c072c80 0x7ff06c097620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:28.969 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.966+0000 7ff071d0e640 1 --2- 172.21.15.105:0/1065999043 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff06c12cf20 0x7ff06c097b60 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7ff05c000c00 tx=0x7ff05c032c10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:28.969 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.966+0000 7ff062ffd640 1 -- 172.21.15.105:0/1065999043 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff05c040070 con 0x7ff06c12cf20 2024-09-21T01:46:28.969 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.966+0000 7ff062ffd640 1 -- 172.21.15.105:0/1065999043 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7ff05c0038e0 con 0x7ff06c12cf20 2024-09-21T01:46:28.969 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.966+0000 7ff074773640 1 -- 172.21.15.105:0/1065999043 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff06c070240 con 0x7ff06c12cf20 2024-09-21T01:46:28.969 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.966+0000 7ff062ffd640 1 -- 172.21.15.105:0/1065999043 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff05c003d30 con 0x7ff06c12cf20 2024-09-21T01:46:28.969 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.966+0000 7ff074773640 1 -- 172.21.15.105:0/1065999043 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff06c1251d0 con 0x7ff06c12cf20 2024-09-21T01:46:28.969 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.967+0000 7ff074773640 1 -- 172.21.15.105:0/1065999043 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff06c126020 con 0x7ff06c12cf20 2024-09-21T01:46:28.970 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.967+0000 7ff062ffd640 1 -- 172.21.15.105:0/1065999043 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7ff05c039040 con 0x7ff06c12cf20 2024-09-21T01:46:28.970 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.968+0000 7ff062ffd640 1 -- 172.21.15.105:0/1065999043 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(30..30 src has 1..30) ==== 3027+0+0 (secure 0 0 0) 0x7ff05c08a7d0 con 0x7ff06c12cf20 2024-09-21T01:46:28.972 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:28.972+0000 7ff062ffd640 1 -- 172.21.15.105:0/1065999043 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7ff06c126020 con 0x7ff06c12cf20 2024-09-21T01:46:29.130 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.130+0000 7ff074773640 1 -- 172.21.15.105:0/1065999043 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd require-osd-release", "release": "squid"} v 0) -- 0x7ff06c12c810 con 0x7ff06c12cf20 2024-09-21T01:46:29.645 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.645+0000 7ff062ffd640 1 -- 172.21.15.105:0/1065999043 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd require-osd-release", "release": "squid"}]=0 v31) ==== 91+0+0 (secure 0 0 0) 0x7ff05c089880 con 0x7ff06c12cf20 2024-09-21T01:46:29.647 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.646+0000 7ff074773640 1 -- 172.21.15.105:0/1065999043 >> v1:172.21.15.105:6816/2838005468 conn(0x7ff0500514b0 legacy=0x7ff0500539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:29.647 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.646+0000 7ff074773640 1 -- 172.21.15.105:0/1065999043 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff06c12cf20 msgr2=0x7ff06c097b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:29.647 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.646+0000 7ff074773640 1 --2- 172.21.15.105:0/1065999043 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff06c12cf20 0x7ff06c097b60 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7ff05c000c00 tx=0x7ff05c032c10 comp rx=0 tx=0).stop 2024-09-21T01:46:29.647 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.646+0000 7ff074773640 1 -- 172.21.15.105:0/1065999043 shutdown_connections 2024-09-21T01:46:29.647 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.646+0000 7ff074773640 1 --2- 172.21.15.105:0/1065999043 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff06c131c50 0x7ff06c124ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:29.647 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.647+0000 7ff074773640 1 --2- 172.21.15.105:0/1065999043 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff06c12cf20 0x7ff06c097b60 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:29.647 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.647+0000 7ff074773640 1 --2- 172.21.15.105:0/1065999043 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff06c072c80 0x7ff06c097620 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:29.648 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.647+0000 7ff074773640 1 -- 172.21.15.105:0/1065999043 >> 172.21.15.105:0/1065999043 conn(0x7ff06c09e270 msgr2=0x7ff06c0983f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:29.648 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.647+0000 7ff074773640 1 -- 172.21.15.105:0/1065999043 shutdown_connections 2024-09-21T01:46:29.648 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.647+0000 7ff074773640 1 -- 172.21.15.105:0/1065999043 wait complete. 2024-09-21T01:46:29.659 DEBUG:teuthology.orchestra.run.smithi105:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'for f in `ceph osd pool ls` ; do ceph osd pool set $f pg_autoscale_mode off ; done' 2024-09-21T01:46:29.768 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.768+0000 7fcd556b3640 1 -- 172.21.15.105:0/2765079194 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fcd500977c0 msgr2=0x7fcd50097bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:29.769 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.768+0000 7fcd556b3640 1 --2- 172.21.15.105:0/2765079194 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fcd500977c0 0x7fcd50097bd0 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7fcd48005fd0 tx=0x7fcd4802a980 comp rx=0 tx=0).stop 2024-09-21T01:46:29.769 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.768+0000 7fcd556b3640 1 -- 172.21.15.105:0/2765079194 shutdown_connections 2024-09-21T01:46:29.769 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.768+0000 7fcd556b3640 1 --2- 172.21.15.105:0/2765079194 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fcd500977c0 0x7fcd50097bd0 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:29.769 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.768+0000 7fcd556b3640 1 --2- 172.21.15.105:0/2765079194 >> v2:172.21.15.105:3300/0 conn(0x7fcd500730f0 0x7fcd50073500 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:29.769 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.768+0000 7fcd556b3640 1 -- 172.21.15.105:0/2765079194 >> 172.21.15.105:0/2765079194 conn(0x7fcd5009e270 msgr2=0x7fcd5009a520 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:29.769 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.768+0000 7fcd556b3640 1 -- 172.21.15.105:0/2765079194 shutdown_connections 2024-09-21T01:46:29.769 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.768+0000 7fcd556b3640 1 -- 172.21.15.105:0/2765079194 wait complete. 2024-09-21T01:46:29.771 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.769+0000 7fcd556b3640 1 Processor -- start 2024-09-21T01:46:29.771 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.769+0000 7fcd556b3640 1 -- start start 2024-09-21T01:46:29.771 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.769+0000 7fcd556b3640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fcd500730f0 0x7fcd5012ac80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:29.771 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.769+0000 7fcd556b3640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fcd50073a40 0x7fcd5012b1c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:29.771 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.769+0000 7fcd556b3640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcd500977c0 0x7fcd5012b700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:29.771 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.769+0000 7fcd556b3640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7fcd50134ab0 con 0x7fcd500977c0 2024-09-21T01:46:29.772 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.769+0000 7fcd556b3640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7fcd50134930 con 0x7fcd500730f0 2024-09-21T01:46:29.772 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.769+0000 7fcd556b3640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7fcd50134c30 con 0x7fcd50073a40 2024-09-21T01:46:29.772 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.769+0000 7fcd4effd640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fcd500730f0 0x7fcd5012ac80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:29.772 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.769+0000 7fcd47fff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fcd50073a40 0x7fcd5012b1c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:29.772 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.769+0000 7fcd4f7fe640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcd500977c0 0x7fcd5012b700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:29.772 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.770+0000 7fcd4effd640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fcd500730f0 0x7fcd5012ac80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:53478/0 (socket says 172.21.15.105:53478) 2024-09-21T01:46:29.772 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.770+0000 7fcd4effd640 1 -- 172.21.15.105:0/4220866747 learned_addr learned my addr 172.21.15.105:0/4220866747 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:29.772 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.770+0000 7fcd47fff640 1 -- 172.21.15.105:0/4220866747 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fcd500730f0 msgr2=0x7fcd5012ac80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:29.772 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.770+0000 7fcd47fff640 1 --2- 172.21.15.105:0/4220866747 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fcd500730f0 0x7fcd5012ac80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:29.772 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.770+0000 7fcd47fff640 1 -- 172.21.15.105:0/4220866747 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcd500977c0 msgr2=0x7fcd5012b700 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:29.772 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.770+0000 7fcd47fff640 1 --2- 172.21.15.105:0/4220866747 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcd500977c0 0x7fcd5012b700 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:29.772 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.770+0000 7fcd47fff640 1 -- 172.21.15.105:0/4220866747 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fcd50098a40 con 0x7fcd50073a40 2024-09-21T01:46:29.772 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.770+0000 7fcd47fff640 1 --2- 172.21.15.105:0/4220866747 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fcd50073a40 0x7fcd5012b1c0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7fcd3800c390 tx=0x7fcd3800c850 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:29.772 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.770+0000 7fcd477fe640 1 -- 172.21.15.105:0/4220866747 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcd38017070 con 0x7fcd50073a40 2024-09-21T01:46:29.773 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.770+0000 7fcd477fe640 1 -- 172.21.15.105:0/4220866747 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fcd380035f0 con 0x7fcd50073a40 2024-09-21T01:46:29.773 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.770+0000 7fcd556b3640 1 -- 172.21.15.105:0/4220866747 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fcd5012bc40 con 0x7fcd50073a40 2024-09-21T01:46:29.773 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.770+0000 7fcd477fe640 1 -- 172.21.15.105:0/4220866747 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fcd38003a40 con 0x7fcd50073a40 2024-09-21T01:46:29.773 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.770+0000 7fcd556b3640 1 -- 172.21.15.105:0/4220866747 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fcd5012c740 con 0x7fcd50073a40 2024-09-21T01:46:29.773 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.771+0000 7fcd556b3640 1 -- 172.21.15.105:0/4220866747 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fcd50073500 con 0x7fcd50073a40 2024-09-21T01:46:29.773 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.771+0000 7fcd477fe640 1 -- 172.21.15.105:0/4220866747 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fcd3801a070 con 0x7fcd50073a40 2024-09-21T01:46:29.774 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.772+0000 7fcd477fe640 1 -- 172.21.15.105:0/4220866747 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(31..31 src has 1..31) ==== 3027+0+0 (secure 0 0 0) 0x7fcd38068270 con 0x7fcd50073a40 2024-09-21T01:46:29.776 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.776+0000 7fcd477fe640 1 -- 172.21.15.105:0/4220866747 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fcd38067320 con 0x7fcd50073a40 2024-09-21T01:46:29.924 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.923+0000 7fcd556b3640 1 -- 172.21.15.105:0/4220866747 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd pool ls"} v 0) -- 0x7fcd5012ca00 con 0x7fcd50073a40 2024-09-21T01:46:29.924 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.923+0000 7fcd477fe640 1 -- 172.21.15.105:0/4220866747 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd pool ls"}]=0 v31) ==== 59+0+9 (secure 0 0 0) 0x7fcd38066e60 con 0x7fcd50073a40 2024-09-21T01:46:29.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.925+0000 7fcd556b3640 1 -- 172.21.15.105:0/4220866747 >> v1:172.21.15.105:6816/2838005468 conn(0x7fcd300514e0 legacy=0x7fcd300539f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:29.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.925+0000 7fcd556b3640 1 -- 172.21.15.105:0/4220866747 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fcd50073a40 msgr2=0x7fcd5012b1c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:29.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.925+0000 7fcd556b3640 1 --2- 172.21.15.105:0/4220866747 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fcd50073a40 0x7fcd5012b1c0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7fcd3800c390 tx=0x7fcd3800c850 comp rx=0 tx=0).stop 2024-09-21T01:46:29.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.925+0000 7fcd556b3640 1 -- 172.21.15.105:0/4220866747 shutdown_connections 2024-09-21T01:46:29.927 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.925+0000 7fcd556b3640 1 --2- 172.21.15.105:0/4220866747 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fcd500977c0 0x7fcd5012b700 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:29.927 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.925+0000 7fcd556b3640 1 --2- 172.21.15.105:0/4220866747 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fcd50073a40 0x7fcd5012b1c0 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:29.927 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.926+0000 7fcd556b3640 1 --2- 172.21.15.105:0/4220866747 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fcd500730f0 0x7fcd5012ac80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:29.927 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.926+0000 7fcd556b3640 1 -- 172.21.15.105:0/4220866747 >> 172.21.15.105:0/4220866747 conn(0x7fcd5009e270 msgr2=0x7fcd5012f9b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:29.927 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.926+0000 7fcd556b3640 1 -- 172.21.15.105:0/4220866747 shutdown_connections 2024-09-21T01:46:29.927 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:29.926+0000 7fcd556b3640 1 -- 172.21.15.105:0/4220866747 wait complete. 2024-09-21T01:46:30.025 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.024+0000 7f7260634640 1 -- 172.21.15.105:0/2762790537 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f725812d0a0 msgr2=0x7f7258128f30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:30.025 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.024+0000 7f7260634640 1 --2- 172.21.15.105:0/2762790537 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f725812d0a0 0x7f7258128f30 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f7248005fd0 tx=0x7f724802a540 comp rx=0 tx=0).stop 2024-09-21T01:46:30.025 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.024+0000 7f7260634640 1 -- 172.21.15.105:0/2762790537 shutdown_connections 2024-09-21T01:46:30.025 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.024+0000 7f7260634640 1 --2- 172.21.15.105:0/2762790537 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f725812d0a0 0x7f7258128f30 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:30.025 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.024+0000 7f7260634640 1 --2- 172.21.15.105:0/2762790537 >> v2:172.21.15.105:3300/0 conn(0x7f725812c750 0x7f725812cb60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:30.025 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.024+0000 7f7260634640 1 -- 172.21.15.105:0/2762790537 >> 172.21.15.105:0/2762790537 conn(0x7f725809b880 msgr2=0x7f725809bc90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:30.026 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.024+0000 7f7260634640 1 -- 172.21.15.105:0/2762790537 shutdown_connections 2024-09-21T01:46:30.026 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.024+0000 7f7260634640 1 -- 172.21.15.105:0/2762790537 wait complete. 2024-09-21T01:46:30.028 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.025+0000 7f7260634640 1 Processor -- start 2024-09-21T01:46:30.028 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.025+0000 7f7260634640 1 -- start start 2024-09-21T01:46:30.028 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.026+0000 7f7260634640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f7258129470 0x7f725806ec00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:30.028 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.026+0000 7f7260634640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f725812c750 0x7f725806f140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:30.028 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.026+0000 7f7260634640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f725812d0a0 0x7f725806f680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:30.028 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.026+0000 7f7260634640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f7258135420 con 0x7f725812d0a0 2024-09-21T01:46:30.028 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.026+0000 7f7260634640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f72581352a0 con 0x7f7258129470 2024-09-21T01:46:30.028 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.026+0000 7f7260634640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f72581355a0 con 0x7f725812c750 2024-09-21T01:46:30.028 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.026+0000 7f725dbcf640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f725812c750 0x7f725806f140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:30.028 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.026+0000 7f725ebd1640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f725812d0a0 0x7f725806f680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:30.028 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.026+0000 7f725e3d0640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f7258129470 0x7f725806ec00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:30.028 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.026+0000 7f725dbcf640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f725812c750 0x7f725806f140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:38828/0 (socket says 172.21.15.105:38828) 2024-09-21T01:46:30.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.026+0000 7f725dbcf640 1 -- 172.21.15.105:0/1241324184 learned_addr learned my addr 172.21.15.105:0/1241324184 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:30.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.026+0000 7f725ebd1640 1 --2- 172.21.15.105:0/1241324184 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f725812d0a0 0x7f725806f680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:34928/0 (socket says 172.21.15.105:34928) 2024-09-21T01:46:30.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.026+0000 7f725dbcf640 1 -- 172.21.15.105:0/1241324184 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f7258129470 msgr2=0x7f725806ec00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:30.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.026+0000 7f725dbcf640 1 --2- 172.21.15.105:0/1241324184 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f7258129470 0x7f725806ec00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:30.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.026+0000 7f725dbcf640 1 -- 172.21.15.105:0/1241324184 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f725812d0a0 msgr2=0x7f725806f680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:30.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.026+0000 7f725dbcf640 1 --2- 172.21.15.105:0/1241324184 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f725812d0a0 0x7f725806f680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:30.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.026+0000 7f725dbcf640 1 -- 172.21.15.105:0/1241324184 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7258074780 con 0x7f725812c750 2024-09-21T01:46:30.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.026+0000 7f725e3d0640 1 --2- 172.21.15.105:0/1241324184 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f7258129470 0x7f725806ec00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:30.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.027+0000 7f725dbcf640 1 --2- 172.21.15.105:0/1241324184 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f725812c750 0x7f725806f140 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f7248034040 tx=0x7f7248005e80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:30.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.027+0000 7f724effd640 1 -- 172.21.15.105:0/1241324184 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f724803e030 con 0x7f725812c750 2024-09-21T01:46:30.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.027+0000 7f724effd640 1 -- 172.21.15.105:0/1241324184 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f7248003c30 con 0x7f725812c750 2024-09-21T01:46:30.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.027+0000 7f7260634640 1 -- 172.21.15.105:0/1241324184 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7258074210 con 0x7f725812c750 2024-09-21T01:46:30.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.027+0000 7f724effd640 1 -- 172.21.15.105:0/1241324184 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7248035040 con 0x7f725812c750 2024-09-21T01:46:30.029 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.027+0000 7f7260634640 1 -- 172.21.15.105:0/1241324184 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f72581c5830 con 0x7f725812c750 2024-09-21T01:46:30.030 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.028+0000 7f7260634640 1 -- 172.21.15.105:0/1241324184 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f725812d9e0 con 0x7f725812c750 2024-09-21T01:46:30.030 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.028+0000 7f724effd640 1 -- 172.21.15.105:0/1241324184 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f7248041030 con 0x7f725812c750 2024-09-21T01:46:30.030 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.028+0000 7f724effd640 1 -- 172.21.15.105:0/1241324184 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(31..31 src has 1..31) ==== 3027+0+0 (secure 0 0 0) 0x7f7248089650 con 0x7f725812c750 2024-09-21T01:46:30.032 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.032+0000 7f724effd640 1 -- 172.21.15.105:0/1241324184 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f725812d9e0 con 0x7f725812c750 2024-09-21T01:46:30.180 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.179+0000 7f7260634640 1 -- 172.21.15.105:0/1241324184 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd pool set", "pool": ".mgr", "var": "pg_autoscale_mode", "val": "off"} v 0) -- 0x7f725812cb60 con 0x7f725812c750 2024-09-21T01:46:30.649 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.649+0000 7f724effd640 1 -- 172.21.15.105:0/1241324184 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_autoscale_mode", "val": "off"}]=0 set pool 1 pg_autoscale_mode to off v32) ==== 153+0+0 (secure 0 0 0) 0x7f7248088700 con 0x7f725812c750 2024-09-21T01:46:30.649 INFO:teuthology.orchestra.run.smithi105.stderr:set pool 1 pg_autoscale_mode to off 2024-09-21T01:46:30.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.652+0000 7f7260634640 1 -- 172.21.15.105:0/1241324184 >> v1:172.21.15.105:6816/2838005468 conn(0x7f723c0514a0 legacy=0x7f723c0539b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:30.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.652+0000 7f7260634640 1 -- 172.21.15.105:0/1241324184 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f725812c750 msgr2=0x7f725806f140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:30.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.652+0000 7f7260634640 1 --2- 172.21.15.105:0/1241324184 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f725812c750 0x7f725806f140 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7f7248034040 tx=0x7f7248005e80 comp rx=0 tx=0).stop 2024-09-21T01:46:30.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.652+0000 7f7260634640 1 -- 172.21.15.105:0/1241324184 shutdown_connections 2024-09-21T01:46:30.652 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.652+0000 7f7260634640 1 --2- 172.21.15.105:0/1241324184 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f725812d0a0 0x7f725806f680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:30.653 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.652+0000 7f7260634640 1 --2- 172.21.15.105:0/1241324184 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f725812c750 0x7f725806f140 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:30.653 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.652+0000 7f7260634640 1 --2- 172.21.15.105:0/1241324184 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f7258129470 0x7f725806ec00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:30.653 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.652+0000 7f7260634640 1 -- 172.21.15.105:0/1241324184 >> 172.21.15.105:0/1241324184 conn(0x7f725809b880 msgr2=0x7f72580995a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:30.653 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.652+0000 7f7260634640 1 -- 172.21.15.105:0/1241324184 shutdown_connections 2024-09-21T01:46:30.653 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.652+0000 7f7260634640 1 -- 172.21.15.105:0/1241324184 wait complete. 2024-09-21T01:46:30.755 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.754+0000 7f7cfc3bc640 1 -- 172.21.15.105:0/1795941237 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f7cf412c800 msgr2=0x7f7cf412cc10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:30.755 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.754+0000 7f7cfc3bc640 1 --2- 172.21.15.105:0/1795941237 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f7cf412c800 0x7f7cf412cc10 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f7cec005fd0 tx=0x7f7cec02a770 comp rx=0 tx=0).stop 2024-09-21T01:46:30.755 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.754+0000 7f7cfc3bc640 1 -- 172.21.15.105:0/1795941237 shutdown_connections 2024-09-21T01:46:30.755 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.754+0000 7f7cfc3bc640 1 --2- 172.21.15.105:0/1795941237 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f7cf412c800 0x7f7cf412cc10 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:30.755 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.754+0000 7f7cfc3bc640 1 --2- 172.21.15.105:0/1795941237 >> v2:172.21.15.105:3300/0 conn(0x7f7cf4072500 0x7f7cf412c2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:30.755 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.754+0000 7f7cfc3bc640 1 --2- 172.21.15.105:0/1795941237 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f7cf4071bb0 0x7f7cf4071fc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:30.755 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.754+0000 7f7cfc3bc640 1 -- 172.21.15.105:0/1795941237 >> 172.21.15.105:0/1795941237 conn(0x7f7cf409bc40 msgr2=0x7f7cf406fd10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:30.755 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.754+0000 7f7cfc3bc640 1 -- 172.21.15.105:0/1795941237 shutdown_connections 2024-09-21T01:46:30.756 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.754+0000 7f7cfc3bc640 1 -- 172.21.15.105:0/1795941237 wait complete. 2024-09-21T01:46:30.758 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.755+0000 7f7cfc3bc640 1 Processor -- start 2024-09-21T01:46:30.758 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.755+0000 7f7cfc3bc640 1 -- start start 2024-09-21T01:46:30.758 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7cfc3bc640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f7cf4071bb0 0x7f7cf4097680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:30.758 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7cfc3bc640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f7cf4072500 0x7f7cf4097bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:30.758 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7cfc3bc640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7cf412c800 0x7f7cf4098100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:30.759 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7cfc3bc640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f7cf4134150 con 0x7f7cf412c800 2024-09-21T01:46:30.759 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7cfc3bc640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f7cf4133fd0 con 0x7f7cf4072500 2024-09-21T01:46:30.759 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7cfc3bc640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f7cf41342d0 con 0x7f7cf4071bb0 2024-09-21T01:46:30.759 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7cfa158640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f7cf4071bb0 0x7f7cf4097680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:30.759 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7cf9957640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f7cf4072500 0x7f7cf4097bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:30.759 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7cfa158640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f7cf4071bb0 0x7f7cf4097680 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:38850/0 (socket says 172.21.15.105:38850) 2024-09-21T01:46:30.759 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7cfa158640 1 -- 172.21.15.105:0/3206406981 learned_addr learned my addr 172.21.15.105:0/3206406981 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:30.759 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7cfa959640 1 --2- 172.21.15.105:0/3206406981 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7cf412c800 0x7f7cf4098100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:30.759 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7cf9957640 1 -- 172.21.15.105:0/3206406981 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f7cf4071bb0 msgr2=0x7f7cf4097680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:30.759 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7cf9957640 1 --2- 172.21.15.105:0/3206406981 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f7cf4071bb0 0x7f7cf4097680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:30.759 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7cf9957640 1 -- 172.21.15.105:0/3206406981 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7cf412c800 msgr2=0x7f7cf4098100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:30.759 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7cf9957640 1 --2- 172.21.15.105:0/3206406981 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7cf412c800 0x7f7cf4098100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:30.760 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7cf9957640 1 -- 172.21.15.105:0/3206406981 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f7cf408c280 con 0x7f7cf4072500 2024-09-21T01:46:30.760 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7cfa158640 1 --2- 172.21.15.105:0/3206406981 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f7cf4071bb0 0x7f7cf4097680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:30.760 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7cf9957640 1 --2- 172.21.15.105:0/3206406981 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f7cf4072500 0x7f7cf4097bc0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f7ce400e9f0 tx=0x7f7ce400eeb0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:30.760 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7ceaffd640 1 -- 172.21.15.105:0/3206406981 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7ce400ccc0 con 0x7f7cf4072500 2024-09-21T01:46:30.760 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7ceaffd640 1 -- 172.21.15.105:0/3206406981 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f7ce400ce60 con 0x7f7cf4072500 2024-09-21T01:46:30.760 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7cfc3bc640 1 -- 172.21.15.105:0/3206406981 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f7cf41325b0 con 0x7f7cf4072500 2024-09-21T01:46:30.760 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7ceaffd640 1 -- 172.21.15.105:0/3206406981 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f7ce4004280 con 0x7f7cf4072500 2024-09-21T01:46:30.760 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.756+0000 7f7cfc3bc640 1 -- 172.21.15.105:0/3206406981 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f7cf41313f0 con 0x7f7cf4072500 2024-09-21T01:46:30.760 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.757+0000 7f7cfc3bc640 1 -- 172.21.15.105:0/3206406981 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f7cf412c2c0 con 0x7f7cf4072500 2024-09-21T01:46:30.760 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.758+0000 7f7ceaffd640 1 -- 172.21.15.105:0/3206406981 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f7ce40035f0 con 0x7f7cf4072500 2024-09-21T01:46:30.760 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.758+0000 7f7ceaffd640 1 -- 172.21.15.105:0/3206406981 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(32..32 src has 1..32) ==== 3027+0+0 (secure 0 0 0) 0x7f7ce40699b0 con 0x7f7cf4072500 2024-09-21T01:46:30.762 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.762+0000 7f7ceaffd640 1 -- 172.21.15.105:0/3206406981 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f7ce4068a60 con 0x7f7cf4072500 2024-09-21T01:46:30.910 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:30.909+0000 7f7cfc3bc640 1 -- 172.21.15.105:0/3206406981 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "osd pool set", "pool": "rbd", "var": "pg_autoscale_mode", "val": "off"} v 0) -- 0x7f7cf4068fc0 con 0x7f7cf4072500 2024-09-21T01:46:31.658 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.658+0000 7f7ceaffd640 1 -- 172.21.15.105:0/3206406981 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{"prefix": "osd pool set", "pool": "rbd", "var": "pg_autoscale_mode", "val": "off"}]=0 set pool 2 pg_autoscale_mode to off v33) ==== 152+0+0 (secure 0 0 0) 0x7f7ce4004420 con 0x7f7cf4072500 2024-09-21T01:46:31.659 INFO:teuthology.orchestra.run.smithi105.stderr:set pool 2 pg_autoscale_mode to off 2024-09-21T01:46:31.661 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.660+0000 7f7cfc3bc640 1 -- 172.21.15.105:0/3206406981 >> v1:172.21.15.105:6816/2838005468 conn(0x7f7cd80514b0 legacy=0x7f7cd80539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:31.661 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.660+0000 7f7cfc3bc640 1 -- 172.21.15.105:0/3206406981 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f7cf4072500 msgr2=0x7f7cf4097bc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:31.661 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.660+0000 7f7cfc3bc640 1 --2- 172.21.15.105:0/3206406981 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f7cf4072500 0x7f7cf4097bc0 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f7ce400e9f0 tx=0x7f7ce400eeb0 comp rx=0 tx=0).stop 2024-09-21T01:46:31.661 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.660+0000 7f7cfc3bc640 1 -- 172.21.15.105:0/3206406981 shutdown_connections 2024-09-21T01:46:31.661 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.660+0000 7f7cfc3bc640 1 --2- 172.21.15.105:0/3206406981 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f7cf412c800 0x7f7cf4098100 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:31.661 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.660+0000 7f7cfc3bc640 1 --2- 172.21.15.105:0/3206406981 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f7cf4072500 0x7f7cf4097bc0 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:31.661 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.660+0000 7f7cfc3bc640 1 --2- 172.21.15.105:0/3206406981 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f7cf4071bb0 0x7f7cf4097680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:31.661 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.660+0000 7f7cfc3bc640 1 -- 172.21.15.105:0/3206406981 >> 172.21.15.105:0/3206406981 conn(0x7f7cf409bc40 msgr2=0x7f7cf412ad40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:31.661 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.660+0000 7f7cfc3bc640 1 -- 172.21.15.105:0/3206406981 shutdown_connections 2024-09-21T01:46:31.661 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.660+0000 7f7cfc3bc640 1 -- 172.21.15.105:0/3206406981 wait complete. 2024-09-21T01:46:31.673 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-21T01:46:31.683 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T01:46:31.683 INFO:tasks.ceph.ceph_manager.ceph:waiting for mgr available 2024-09-21T01:46:31.683 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph mgr dump --format=json 2024-09-21T01:46:31.795 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.794+0000 7f9537981640 1 -- 172.21.15.105:0/2279628051 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f953012de50 msgr2=0x7f9530129ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:31.795 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.794+0000 7f9537981640 1 --2- 172.21.15.105:0/2279628051 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f953012de50 0x7f9530129ec0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f9520009830 tx=0x7f952002a3e0 comp rx=0 tx=0).stop 2024-09-21T01:46:31.795 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.794+0000 7f9537981640 1 -- 172.21.15.105:0/2279628051 shutdown_connections 2024-09-21T01:46:31.795 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.794+0000 7f9537981640 1 --2- 172.21.15.105:0/2279628051 >> v2:172.21.15.105:3300/0 conn(0x7f953012a400 0x7f953012a810 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:31.795 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.794+0000 7f9537981640 1 --2- 172.21.15.105:0/2279628051 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f953012de50 0x7f9530129ec0 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:31.795 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.794+0000 7f9537981640 1 --2- 172.21.15.105:0/2279628051 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9530073b30 0x7f953012d910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:31.795 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.794+0000 7f9537981640 1 -- 172.21.15.105:0/2279628051 >> 172.21.15.105:0/2279628051 conn(0x7f953009fe70 msgr2=0x7f95300a0280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:31.795 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.794+0000 7f9537981640 1 -- 172.21.15.105:0/2279628051 shutdown_connections 2024-09-21T01:46:31.796 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.794+0000 7f9537981640 1 -- 172.21.15.105:0/2279628051 wait complete. 2024-09-21T01:46:31.797 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.795+0000 7f9537981640 1 Processor -- start 2024-09-21T01:46:31.797 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.795+0000 7f9537981640 1 -- start start 2024-09-21T01:46:31.798 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.796+0000 7f9537981640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9530073b30 0x7f95300989e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:31.798 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.796+0000 7f9537981640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f953012a400 0x7f9530098f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:31.798 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.796+0000 7f9537981640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f953012de50 0x7f9530099460 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:31.798 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.796+0000 7f9537981640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f9530135280 con 0x7f953012a400 2024-09-21T01:46:31.798 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.796+0000 7f9537981640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f9530135100 con 0x7f9530073b30 2024-09-21T01:46:31.798 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.796+0000 7f9537981640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f9530135400 con 0x7f953012de50 2024-09-21T01:46:31.798 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.796+0000 7f953571d640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9530073b30 0x7f95300989e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:31.798 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.796+0000 7f9534f1c640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f953012a400 0x7f9530098f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:31.798 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.796+0000 7f953571d640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9530073b30 0x7f95300989e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:50484/0 (socket says 172.21.15.105:50484) 2024-09-21T01:46:31.798 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.796+0000 7f953571d640 1 -- 172.21.15.105:0/3637568172 learned_addr learned my addr 172.21.15.105:0/3637568172 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:31.798 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.796+0000 7f9535f1e640 1 --2- 172.21.15.105:0/3637568172 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f953012de50 0x7f9530099460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:31.798 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.796+0000 7f953571d640 1 -- 172.21.15.105:0/3637568172 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f953012de50 msgr2=0x7f9530099460 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:31.799 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.796+0000 7f953571d640 1 --2- 172.21.15.105:0/3637568172 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f953012de50 0x7f9530099460 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:31.799 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.796+0000 7f953571d640 1 -- 172.21.15.105:0/3637568172 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f953012a400 msgr2=0x7f9530098f20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:31.799 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.796+0000 7f953571d640 1 --2- 172.21.15.105:0/3637568172 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f953012a400 0x7f9530098f20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:31.799 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.796+0000 7f953571d640 1 -- 172.21.15.105:0/3637568172 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f9530134260 con 0x7f9530073b30 2024-09-21T01:46:31.799 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.796+0000 7f953571d640 1 --2- 172.21.15.105:0/3637568172 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9530073b30 0x7f95300989e0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f952c00ef60 tx=0x7f952c00ef90 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:31.799 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.797+0000 7f9525ffb640 1 -- 172.21.15.105:0/3637568172 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f952c003970 con 0x7f9530073b30 2024-09-21T01:46:31.799 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.797+0000 7f9525ffb640 1 -- 172.21.15.105:0/3637568172 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f952c003c00 con 0x7f9530073b30 2024-09-21T01:46:31.799 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.797+0000 7f9537981640 1 -- 172.21.15.105:0/3637568172 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f9530133a60 con 0x7f9530073b30 2024-09-21T01:46:31.799 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.797+0000 7f9525ffb640 1 -- 172.21.15.105:0/3637568172 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f952c004020 con 0x7f9530073b30 2024-09-21T01:46:31.799 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.797+0000 7f9537981640 1 -- 172.21.15.105:0/3637568172 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f9530131b90 con 0x7f9530073b30 2024-09-21T01:46:31.800 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.798+0000 7f9537981640 1 -- 172.21.15.105:0/3637568172 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f9530132b20 con 0x7f9530073b30 2024-09-21T01:46:31.800 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.798+0000 7f9525ffb640 1 -- 172.21.15.105:0/3637568172 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f952c01d020 con 0x7f9530073b30 2024-09-21T01:46:31.800 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.798+0000 7f9525ffb640 1 -- 172.21.15.105:0/3637568172 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f952c069480 con 0x7f9530073b30 2024-09-21T01:46:31.802 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.802+0000 7f9525ffb640 1 -- 172.21.15.105:0/3637568172 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f952c068530 con 0x7f9530073b30 2024-09-21T01:46:31.981 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.981+0000 7f9537981640 1 -- 172.21.15.105:0/3637568172 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "mgr dump", "format": "json"} v 0) -- 0x7f953012a810 con 0x7f9530073b30 2024-09-21T01:46:31.983 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.983+0000 7f9525ffb640 1 -- 172.21.15.105:0/3637568172 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json"}]=0 v10) ==== 74+0+124224 (secure 0 0 0) 0x7f952c009780 con 0x7f9530073b30 2024-09-21T01:46:31.984 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:31.986 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.985+0000 7f9537981640 1 -- 172.21.15.105:0/3637568172 >> v1:172.21.15.105:6816/2838005468 conn(0x7f95140514b0 legacy=0x7f95140539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:31.986 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.985+0000 7f9537981640 1 -- 172.21.15.105:0/3637568172 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9530073b30 msgr2=0x7f95300989e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:31.986 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.985+0000 7f9537981640 1 --2- 172.21.15.105:0/3637568172 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9530073b30 0x7f95300989e0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7f952c00ef60 tx=0x7f952c00ef90 comp rx=0 tx=0).stop 2024-09-21T01:46:31.986 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.985+0000 7f9537981640 1 -- 172.21.15.105:0/3637568172 shutdown_connections 2024-09-21T01:46:31.986 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.985+0000 7f9537981640 1 --2- 172.21.15.105:0/3637568172 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f953012de50 0x7f9530099460 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:31.986 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.985+0000 7f9537981640 1 --2- 172.21.15.105:0/3637568172 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f953012a400 0x7f9530098f20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:31.986 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.985+0000 7f9537981640 1 --2- 172.21.15.105:0/3637568172 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f9530073b30 0x7f95300989e0 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:31.986 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.985+0000 7f9537981640 1 -- 172.21.15.105:0/3637568172 >> 172.21.15.105:0/3637568172 conn(0x7f953009fe70 msgr2=0x7f953012c090 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:31.986 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.985+0000 7f9537981640 1 -- 172.21.15.105:0/3637568172 shutdown_connections 2024-09-21T01:46:31.986 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:31.985+0000 7f9537981640 1 -- 172.21.15.105:0/3637568172 wait complete. 2024-09-21T01:46:31.997 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":10,"flags":0,"active_gid":14125,"active_name":"y","active_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6816","nonce":2838005468}]},"active_addr":"172.21.15.105:6816/2838005468","active_change":"2024-09-21T01:46:13.485232+0000","active_mgr_features":4540719139924082687,"available":true,"standbys":[{"gid":14132,"name":"x","mgr_features":4540719139924082687,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":20,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.105:0","nonce":2925579744}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.105:0","nonce":79562422}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.105:0","nonce":2871062666}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.105:0","nonce":854224109}]}]} 2024-09-21T01:46:31.999 INFO:tasks.ceph.ceph_manager.ceph:mgr available! 2024-09-21T01:46:31.999 INFO:tasks.ceph.ceph_manager.ceph:waiting for all up 2024-09-21T01:46:32.000 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T01:46:32.111 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.110+0000 7fd8c0c61640 1 -- 172.21.15.105:0/891555502 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd8bc12c440 msgr2=0x7fd8bc132e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.111 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.110+0000 7fd8c0c61640 1 --2- 172.21.15.105:0/891555502 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd8bc12c440 0x7fd8bc132e60 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7fd8b4005fd0 tx=0x7fd8b402a770 comp rx=0 tx=0).stop 2024-09-21T01:46:32.111 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.110+0000 7fd8c0c61640 1 -- 172.21.15.105:0/891555502 shutdown_connections 2024-09-21T01:46:32.111 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.110+0000 7fd8c0c61640 1 --2- 172.21.15.105:0/891555502 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd8bc12c440 0x7fd8bc132e60 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.111 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.110+0000 7fd8c0c61640 1 --2- 172.21.15.105:0/891555502 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd8bc12baf0 0x7fd8bc12bf00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.111 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.110+0000 7fd8c0c61640 1 --2- 172.21.15.105:0/891555502 >> v2:172.21.15.105:3300/0 conn(0x7fd8bc0733e0 0x7fd8bc0737f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.111 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.110+0000 7fd8c0c61640 1 -- 172.21.15.105:0/891555502 >> 172.21.15.105:0/891555502 conn(0x7fd8bc09b7c0 msgr2=0x7fd8bc09bbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:32.111 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.110+0000 7fd8c0c61640 1 -- 172.21.15.105:0/891555502 shutdown_connections 2024-09-21T01:46:32.111 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.110+0000 7fd8c0c61640 1 -- 172.21.15.105:0/891555502 wait complete. 2024-09-21T01:46:32.112 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.111+0000 7fd8c0c61640 1 Processor -- start 2024-09-21T01:46:32.112 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.112+0000 7fd8c0c61640 1 -- start start 2024-09-21T01:46:32.114 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.112+0000 7fd8c0c61640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd8bc0733e0 0x7fd8bc131eb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.114 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.112+0000 7fd8c0c61640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd8bc12baf0 0x7fd8bc1323f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.114 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.112+0000 7fd8c0c61640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd8bc12c440 0x7fd8bc132930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.114 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.112+0000 7fd8c0c61640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7fd8bc06fbc0 con 0x7fd8bc12baf0 2024-09-21T01:46:32.114 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.112+0000 7fd8c0c61640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7fd8bc06fa40 con 0x7fd8bc12c440 2024-09-21T01:46:32.114 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.112+0000 7fd8c0c61640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7fd8bc1333a0 con 0x7fd8bc0733e0 2024-09-21T01:46:32.114 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.112+0000 7fd8ba59c640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd8bc0733e0 0x7fd8bc131eb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.114 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.112+0000 7fd8b1d9b640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd8bc12baf0 0x7fd8bc1323f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.114 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.112+0000 7fd8bad9d640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd8bc12c440 0x7fd8bc132930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.114 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.112+0000 7fd8ba59c640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd8bc0733e0 0x7fd8bc131eb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:48294/0 (socket says 172.21.15.105:48294) 2024-09-21T01:46:32.114 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.112+0000 7fd8ba59c640 1 -- 172.21.15.105:0/433486764 learned_addr learned my addr 172.21.15.105:0/433486764 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:32.114 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.113+0000 7fd8b1d9b640 1 -- 172.21.15.105:0/433486764 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd8bc0733e0 msgr2=0x7fd8bc131eb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.115 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.113+0000 7fd8b1d9b640 1 --2- 172.21.15.105:0/433486764 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd8bc0733e0 0x7fd8bc131eb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.115 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.113+0000 7fd8b1d9b640 1 -- 172.21.15.105:0/433486764 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd8bc12c440 msgr2=0x7fd8bc132930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.115 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.113+0000 7fd8b1d9b640 1 --2- 172.21.15.105:0/433486764 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd8bc12c440 0x7fd8bc132930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.115 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.113+0000 7fd8b1d9b640 1 -- 172.21.15.105:0/433486764 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd8bc08d520 con 0x7fd8bc12baf0 2024-09-21T01:46:32.115 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.113+0000 7fd8ba59c640 1 --2- 172.21.15.105:0/433486764 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd8bc0733e0 0x7fd8bc131eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:32.115 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.113+0000 7fd8bad9d640 1 --2- 172.21.15.105:0/433486764 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd8bc12c440 0x7fd8bc132930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:32.115 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.113+0000 7fd8b1d9b640 1 --2- 172.21.15.105:0/433486764 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd8bc12baf0 0x7fd8bc1323f0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7fd8a400ea90 tx=0x7fd8a400ef50 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:32.115 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.113+0000 7fd8b37fe640 1 -- 172.21.15.105:0/433486764 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd8a401b070 con 0x7fd8bc12baf0 2024-09-21T01:46:32.115 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.113+0000 7fd8b37fe640 1 -- 172.21.15.105:0/433486764 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fd8a400cd70 con 0x7fd8bc12baf0 2024-09-21T01:46:32.115 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.113+0000 7fd8c0c61640 1 -- 172.21.15.105:0/433486764 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd8bc08cf70 con 0x7fd8bc12baf0 2024-09-21T01:46:32.115 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.113+0000 7fd8b37fe640 1 -- 172.21.15.105:0/433486764 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd8a4016670 con 0x7fd8bc12baf0 2024-09-21T01:46:32.116 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.113+0000 7fd8c0c61640 1 -- 172.21.15.105:0/433486764 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fd8bc1c4800 con 0x7fd8bc12baf0 2024-09-21T01:46:32.116 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.114+0000 7fd8b37fe640 1 -- 172.21.15.105:0/433486764 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fd8a401e070 con 0x7fd8bc12baf0 2024-09-21T01:46:32.116 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.114+0000 7fd8c0c61640 1 -- 172.21.15.105:0/433486764 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd8bc0737f0 con 0x7fd8bc12baf0 2024-09-21T01:46:32.116 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.115+0000 7fd8b37fe640 1 -- 172.21.15.105:0/433486764 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fd8a40693d0 con 0x7fd8bc12baf0 2024-09-21T01:46:32.118 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.118+0000 7fd8b37fe640 1 -- 172.21.15.105:0/433486764 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fd8a4068480 con 0x7fd8bc12baf0 2024-09-21T01:46:32.268 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.268+0000 7fd8c0c61640 1 -- 172.21.15.105:0/433486764 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7fd8bc069ce0 con 0x7fd8bc12baf0 2024-09-21T01:46:32.269 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.269+0000 7fd8b37fe640 1 -- 172.21.15.105:0/433486764 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v33) ==== 74+0+9196 (secure 0 0 0) 0x7fd8a4067fc0 con 0x7fd8bc12baf0 2024-09-21T01:46:32.269 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:32.269 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":33,"fsid":"355399d3-6490-4ad0-ac52-d793648ff276","created":"2024-09-21T01:44:37.282024+0000","modified":"2024-09-21T01:46:31.651379+0000","last_up_change":"2024-09-21T01:46:27.619214+0000","last_in_change":"2024-09-21T01:44:39.666350+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":7,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"reef","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T01:44:50.380956+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"32","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T01:44:54.210449+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"03218bdb-ec48-44c7-bbf5-31cb53a3cea7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":20,"up_from":26,"up_thru":26,"down_at":21,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6800","nonce":249246859}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6801","nonce":249246859}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6803","nonce":249246859}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6802","nonce":249246859}]},"public_addr":"172.21.15.105:6800/249246859","cluster_addr":"172.21.15.105:6801/249246859","heartbeat_back_addr":"172.21.15.105:6803/249246859","heartbeat_front_addr":"172.21.15.105:6802/249246859","state":["exists","up"]},{"osd":1,"uuid":"5528ff44-922a-4e44-bb69-95fd9544d8dd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":20,"up_from":27,"up_thru":27,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6804","nonce":1896987326}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6805","nonce":1896987326}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6807","nonce":1896987326}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6806","nonce":1896987326}]},"public_addr":"172.21.15.105:6804/1896987326","cluster_addr":"172.21.15.105:6805/1896987326","heartbeat_back_addr":"172.21.15.105:6807/1896987326","heartbeat_front_addr":"172.21.15.105:6806/1896987326","state":["exists","up"]},{"osd":2,"uuid":"df56ff35-71e9-4d46-8abb-84265eb6e1ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":22,"up_from":29,"up_thru":29,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6808","nonce":2933198036}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6809","nonce":2933198036}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6811","nonce":2933198036}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6810","nonce":2933198036}]},"public_addr":"172.21.15.105:6808/2933198036","cluster_addr":"172.21.15.105:6809/2933198036","heartbeat_back_addr":"172.21.15.105:6811/2933198036","heartbeat_front_addr":"172.21.15.105:6810/2933198036","state":["exists","up"]},{"osd":3,"uuid":"012d9668-f1d3-4268-b0eb-15fa845ef17c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":23,"up_from":29,"up_thru":29,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6812","nonce":568702146}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6813","nonce":568702146}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6815","nonce":568702146}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6814","nonce":568702146}]},"public_addr":"172.21.15.105:6812/568702146","cluster_addr":"172.21.15.105:6813/568702146","heartbeat_back_addr":"172.21.15.105:6815/568702146","heartbeat_front_addr":"172.21.15.105:6814/568702146","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.554795+0000","dead_epoch":21},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.869400+0000","dead_epoch":22},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.601687+0000","dead_epoch":23},{"osd":3,"down_stamp":"2024-09-21T01:46:21.001569+0000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.822680+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.105:0/3561680891":"2024-09-22T01:46:13.485080+0000","172.21.15.105:0/2570087590":"2024-09-22T01:46:13.485080+0000","172.21.15.105:0/3837276043":"2024-09-22T01:46:13.485080+0000","172.21.15.105:0/2810478014":"2024-09-22T01:46:13.485080+0000","172.21.15.105:6816/3933313069":"2024-09-22T01:46:13.485080+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T01:46:32.271 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.270+0000 7fd8c0c61640 1 -- 172.21.15.105:0/433486764 >> v1:172.21.15.105:6816/2838005468 conn(0x7fd89c0514b0 legacy=0x7fd89c0539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.271 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.270+0000 7fd8c0c61640 1 -- 172.21.15.105:0/433486764 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd8bc12baf0 msgr2=0x7fd8bc1323f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.271 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.270+0000 7fd8c0c61640 1 --2- 172.21.15.105:0/433486764 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd8bc12baf0 0x7fd8bc1323f0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7fd8a400ea90 tx=0x7fd8a400ef50 comp rx=0 tx=0).stop 2024-09-21T01:46:32.271 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.271+0000 7fd8c0c61640 1 -- 172.21.15.105:0/433486764 shutdown_connections 2024-09-21T01:46:32.272 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.271+0000 7fd8c0c61640 1 --2- 172.21.15.105:0/433486764 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd8bc12c440 0x7fd8bc132930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.272 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.271+0000 7fd8c0c61640 1 --2- 172.21.15.105:0/433486764 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd8bc12baf0 0x7fd8bc1323f0 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.272 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.271+0000 7fd8c0c61640 1 --2- 172.21.15.105:0/433486764 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd8bc0733e0 0x7fd8bc131eb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.272 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.271+0000 7fd8c0c61640 1 -- 172.21.15.105:0/433486764 >> 172.21.15.105:0/433486764 conn(0x7fd8bc09b7c0 msgr2=0x7fd8bc097710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:32.272 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.271+0000 7fd8c0c61640 1 -- 172.21.15.105:0/433486764 shutdown_connections 2024-09-21T01:46:32.272 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.271+0000 7fd8c0c61640 1 -- 172.21.15.105:0/433486764 wait complete. 2024-09-21T01:46:32.283 INFO:tasks.ceph.ceph_manager.ceph:all up! 2024-09-21T01:46:32.283 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T01:46:32.392 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.392+0000 7f400b69b640 1 -- 172.21.15.105:0/3287952171 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4004072d10 msgr2=0x7f4004073120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.393 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.392+0000 7f400b69b640 1 --2- 172.21.15.105:0/3287952171 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4004072d10 0x7f4004073120 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f4000009830 tx=0x7f400002a3f0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.393 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.392+0000 7f400b69b640 1 -- 172.21.15.105:0/3287952171 shutdown_connections 2024-09-21T01:46:32.393 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.392+0000 7f400b69b640 1 --2- 172.21.15.105:0/3287952171 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4004099960 0x7f4004135e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.393 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.392+0000 7f400b69b640 1 --2- 172.21.15.105:0/3287952171 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4004072d10 0x7f4004073120 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.393 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.392+0000 7f400b69b640 1 -- 172.21.15.105:0/3287952171 >> 172.21.15.105:0/3287952171 conn(0x7f40040a01d0 msgr2=0x7f400409c6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:32.393 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.392+0000 7f400b69b640 1 -- 172.21.15.105:0/3287952171 shutdown_connections 2024-09-21T01:46:32.393 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.392+0000 7f400b69b640 1 -- 172.21.15.105:0/3287952171 wait complete. 2024-09-21T01:46:32.395 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.393+0000 7f400b69b640 1 Processor -- start 2024-09-21T01:46:32.395 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.393+0000 7f400b69b640 1 -- start start 2024-09-21T01:46:32.395 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.393+0000 7f400b69b640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4004072d10 0x7f400406faf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.395 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.393+0000 7f400b69b640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4004073660 0x7f4004070030 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.396 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.393+0000 7f400b69b640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4004099960 0x7f4004070570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.396 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.393+0000 7f400b69b640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f4004136810 con 0x7f4004072d10 2024-09-21T01:46:32.396 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.393+0000 7f400b69b640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f4004136690 con 0x7f4004099960 2024-09-21T01:46:32.396 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.393+0000 7f400b69b640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f4004136990 con 0x7f4004073660 2024-09-21T01:46:32.396 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.394+0000 7f4009437640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4004072d10 0x7f400406faf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.396 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.394+0000 7f4009437640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4004072d10 0x7f400406faf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:52076/0 (socket says 172.21.15.105:52076) 2024-09-21T01:46:32.396 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.394+0000 7f4009437640 1 -- 172.21.15.105:0/2568893281 learned_addr learned my addr 172.21.15.105:0/2568893281 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:32.396 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.394+0000 7f4009c38640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4004099960 0x7f4004070570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.396 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.394+0000 7f4008c36640 1 --2- 172.21.15.105:0/2568893281 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4004073660 0x7f4004070030 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.396 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.394+0000 7f4009c38640 1 -- 172.21.15.105:0/2568893281 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4004073660 msgr2=0x7f4004070030 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.396 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.394+0000 7f4009c38640 1 --2- 172.21.15.105:0/2568893281 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4004073660 0x7f4004070030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.396 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.394+0000 7f4009c38640 1 -- 172.21.15.105:0/2568893281 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4004072d10 msgr2=0x7f400406faf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.396 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.394+0000 7f4009c38640 1 --2- 172.21.15.105:0/2568893281 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4004072d10 0x7f400406faf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.396 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.394+0000 7f4009c38640 1 -- 172.21.15.105:0/2568893281 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f4004070db0 con 0x7f4004099960 2024-09-21T01:46:32.397 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.394+0000 7f4009437640 1 --2- 172.21.15.105:0/2568893281 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4004072d10 0x7f400406faf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:32.397 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.394+0000 7f4009c38640 1 --2- 172.21.15.105:0/2568893281 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4004099960 0x7f4004070570 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f3ffc009fd0 tx=0x7f3ffc00c740 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:32.397 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.394+0000 7f3ff9ffb640 1 -- 172.21.15.105:0/2568893281 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3ffc003970 con 0x7f4004099960 2024-09-21T01:46:32.397 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.394+0000 7f3ff9ffb640 1 -- 172.21.15.105:0/2568893281 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f3ffc003c00 con 0x7f4004099960 2024-09-21T01:46:32.397 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.394+0000 7f400b69b640 1 -- 172.21.15.105:0/2568893281 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f4004127f80 con 0x7f4004099960 2024-09-21T01:46:32.397 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.394+0000 7f3ff9ffb640 1 -- 172.21.15.105:0/2568893281 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f3ffc004020 con 0x7f4004099960 2024-09-21T01:46:32.397 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.394+0000 7f400b69b640 1 -- 172.21.15.105:0/2568893281 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f40041351c0 con 0x7f4004099960 2024-09-21T01:46:32.397 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.396+0000 7f3ff9ffb640 1 -- 172.21.15.105:0/2568893281 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f3ffc01d020 con 0x7f4004099960 2024-09-21T01:46:32.397 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.396+0000 7f400b69b640 1 -- 172.21.15.105:0/2568893281 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f4004073120 con 0x7f4004099960 2024-09-21T01:46:32.398 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.396+0000 7f3ff9ffb640 1 -- 172.21.15.105:0/2568893281 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f3ffc069410 con 0x7f4004099960 2024-09-21T01:46:32.402 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.400+0000 7f3ff9ffb640 1 -- 172.21.15.105:0/2568893281 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f4004127f80 con 0x7f4004099960 2024-09-21T01:46:32.548 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.548+0000 7f400b69b640 1 -- 172.21.15.105:0/2568893281 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7f400406a690 con 0x7f4004099960 2024-09-21T01:46:32.549 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.548+0000 7f3ff9ffb640 1 -- 172.21.15.105:0/2568893281 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v33) ==== 74+0+9196 (secure 0 0 0) 0x7f3ffc0684c0 con 0x7f4004099960 2024-09-21T01:46:32.549 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:32.549 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":33,"fsid":"355399d3-6490-4ad0-ac52-d793648ff276","created":"2024-09-21T01:44:37.282024+0000","modified":"2024-09-21T01:46:31.651379+0000","last_up_change":"2024-09-21T01:46:27.619214+0000","last_in_change":"2024-09-21T01:44:39.666350+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":7,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"reef","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T01:44:50.380956+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"32","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T01:44:54.210449+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"03218bdb-ec48-44c7-bbf5-31cb53a3cea7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":20,"up_from":26,"up_thru":26,"down_at":21,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6800","nonce":249246859}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6801","nonce":249246859}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6803","nonce":249246859}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6802","nonce":249246859}]},"public_addr":"172.21.15.105:6800/249246859","cluster_addr":"172.21.15.105:6801/249246859","heartbeat_back_addr":"172.21.15.105:6803/249246859","heartbeat_front_addr":"172.21.15.105:6802/249246859","state":["exists","up"]},{"osd":1,"uuid":"5528ff44-922a-4e44-bb69-95fd9544d8dd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":20,"up_from":27,"up_thru":27,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6804","nonce":1896987326}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6805","nonce":1896987326}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6807","nonce":1896987326}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6806","nonce":1896987326}]},"public_addr":"172.21.15.105:6804/1896987326","cluster_addr":"172.21.15.105:6805/1896987326","heartbeat_back_addr":"172.21.15.105:6807/1896987326","heartbeat_front_addr":"172.21.15.105:6806/1896987326","state":["exists","up"]},{"osd":2,"uuid":"df56ff35-71e9-4d46-8abb-84265eb6e1ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":22,"up_from":29,"up_thru":29,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6808","nonce":2933198036}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6809","nonce":2933198036}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6811","nonce":2933198036}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6810","nonce":2933198036}]},"public_addr":"172.21.15.105:6808/2933198036","cluster_addr":"172.21.15.105:6809/2933198036","heartbeat_back_addr":"172.21.15.105:6811/2933198036","heartbeat_front_addr":"172.21.15.105:6810/2933198036","state":["exists","up"]},{"osd":3,"uuid":"012d9668-f1d3-4268-b0eb-15fa845ef17c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":23,"up_from":29,"up_thru":29,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6812","nonce":568702146}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6813","nonce":568702146}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6815","nonce":568702146}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6814","nonce":568702146}]},"public_addr":"172.21.15.105:6812/568702146","cluster_addr":"172.21.15.105:6813/568702146","heartbeat_back_addr":"172.21.15.105:6815/568702146","heartbeat_front_addr":"172.21.15.105:6814/568702146","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.554795+0000","dead_epoch":21},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.869400+0000","dead_epoch":22},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.601687+0000","dead_epoch":23},{"osd":3,"down_stamp":"2024-09-21T01:46:21.001569+0000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.822680+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.105:0/3561680891":"2024-09-22T01:46:13.485080+0000","172.21.15.105:0/2570087590":"2024-09-22T01:46:13.485080+0000","172.21.15.105:0/3837276043":"2024-09-22T01:46:13.485080+0000","172.21.15.105:0/2810478014":"2024-09-22T01:46:13.485080+0000","172.21.15.105:6816/3933313069":"2024-09-22T01:46:13.485080+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T01:46:32.551 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.550+0000 7f400b69b640 1 -- 172.21.15.105:0/2568893281 >> v1:172.21.15.105:6816/2838005468 conn(0x7f3fe80514b0 legacy=0x7f3fe80539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.551 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.550+0000 7f400b69b640 1 -- 172.21.15.105:0/2568893281 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4004099960 msgr2=0x7f4004070570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.551 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.550+0000 7f400b69b640 1 --2- 172.21.15.105:0/2568893281 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4004099960 0x7f4004070570 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7f3ffc009fd0 tx=0x7f3ffc00c740 comp rx=0 tx=0).stop 2024-09-21T01:46:32.551 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.551+0000 7f400b69b640 1 -- 172.21.15.105:0/2568893281 shutdown_connections 2024-09-21T01:46:32.551 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.551+0000 7f400b69b640 1 --2- 172.21.15.105:0/2568893281 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f4004099960 0x7f4004070570 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.551 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.551+0000 7f400b69b640 1 --2- 172.21.15.105:0/2568893281 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f4004073660 0x7f4004070030 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.552 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.551+0000 7f400b69b640 1 --2- 172.21.15.105:0/2568893281 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f4004072d10 0x7f400406faf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.552 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.551+0000 7f400b69b640 1 -- 172.21.15.105:0/2568893281 >> 172.21.15.105:0/2568893281 conn(0x7f40040a01d0 msgr2=0x7f400412d2b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:32.552 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.551+0000 7f400b69b640 1 -- 172.21.15.105:0/2568893281 shutdown_connections 2024-09-21T01:46:32.552 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.551+0000 7f400b69b640 1 -- 172.21.15.105:0/2568893281 wait complete. 2024-09-21T01:46:32.563 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 flush_pg_stats 2024-09-21T01:46:32.563 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 flush_pg_stats 2024-09-21T01:46:32.563 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 flush_pg_stats 2024-09-21T01:46:32.564 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 flush_pg_stats 2024-09-21T01:46:32.687 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.686+0000 7ff9582a9640 1 -- 172.21.15.105:0/518664176 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff950072380 msgr2=0x7ff950131970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.687 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.686+0000 7ff9582a9640 1 --2- 172.21.15.105:0/518664176 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff950072380 0x7ff950131970 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7ff94000fb10 tx=0x7ff94002e740 comp rx=0 tx=0).stop 2024-09-21T01:46:32.687 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.686+0000 7fe47250d640 1 -- 172.21.15.105:0/4196347125 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe46c12f770 msgr2=0x7fe46c071730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.687 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.686+0000 7fe47250d640 1 --2- 172.21.15.105:0/4196347125 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe46c12f770 0x7fe46c071730 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7fe45c005fd0 tx=0x7fe45c02a540 comp rx=0 tx=0).stop 2024-09-21T01:46:32.688 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.686+0000 7ff9582a9640 1 -- 172.21.15.105:0/518664176 shutdown_connections 2024-09-21T01:46:32.688 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.686+0000 7ff9582a9640 1 --2- 172.21.15.105:0/518664176 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff950131eb0 0x7ff9501342d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.688 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.686+0000 7ff9582a9640 1 --2- 172.21.15.105:0/518664176 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff950072380 0x7ff950131970 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.688 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.686+0000 7ff9582a9640 1 -- 172.21.15.105:0/518664176 >> 172.21.15.105:0/518664176 conn(0x7ff95009f000 msgr2=0x7ff95009b4d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:32.688 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.686+0000 7fe47250d640 1 -- 172.21.15.105:0/4196347125 shutdown_connections 2024-09-21T01:46:32.688 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.686+0000 7fe47250d640 1 --2- 172.21.15.105:0/4196347125 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fe46c071060 0x7fe46c12c770 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.688 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.686+0000 7fe47250d640 1 --2- 172.21.15.105:0/4196347125 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe46c12f770 0x7fe46c071730 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.688 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.686+0000 7fe47250d640 1 -- 172.21.15.105:0/4196347125 >> 172.21.15.105:0/4196347125 conn(0x7fe46c0a2080 msgr2=0x7fe46c0a2490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:32.689 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.687+0000 7ff9582a9640 1 -- 172.21.15.105:0/518664176 shutdown_connections 2024-09-21T01:46:32.689 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.687+0000 7ff9582a9640 1 -- 172.21.15.105:0/518664176 wait complete. 2024-09-21T01:46:32.689 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.687+0000 7fe47250d640 1 -- 172.21.15.105:0/4196347125 shutdown_connections 2024-09-21T01:46:32.689 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.687+0000 7fe47250d640 1 -- 172.21.15.105:0/4196347125 wait complete. 2024-09-21T01:46:32.690 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.687+0000 7f228942d640 1 -- 172.21.15.105:0/68035554 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2284131e60 msgr2=0x7f2284134280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.691 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.687+0000 7f228942d640 1 --2- 172.21.15.105:0/68035554 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2284131e60 0x7f2284134280 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f227c009be0 tx=0x7f227c02a800 comp rx=0 tx=0).stop 2024-09-21T01:46:32.691 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.687+0000 7f228942d640 1 -- 172.21.15.105:0/68035554 shutdown_connections 2024-09-21T01:46:32.691 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.687+0000 7f228942d640 1 --2- 172.21.15.105:0/68035554 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2284131e60 0x7f2284134280 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.691 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.687+0000 7f228942d640 1 --2- 172.21.15.105:0/68035554 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2284097650 0x7f2284131920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.691 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.687+0000 7f228942d640 1 -- 172.21.15.105:0/68035554 >> 172.21.15.105:0/68035554 conn(0x7f228409e4e0 msgr2=0x7f228409e8f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:32.691 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.687+0000 7f228942d640 1 -- 172.21.15.105:0/68035554 shutdown_connections 2024-09-21T01:46:32.691 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.687+0000 7f228942d640 1 -- 172.21.15.105:0/68035554 wait complete. 2024-09-21T01:46:32.692 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.687+0000 7f676cf4d640 1 -- 172.21.15.105:0/647520017 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f67680721b0 msgr2=0x7f67680980b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.692 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.687+0000 7f676cf4d640 1 --2- 172.21.15.105:0/647520017 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f67680721b0 0x7f67680980b0 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f675c009830 tx=0x7f675c02a510 comp rx=0 tx=0).stop 2024-09-21T01:46:32.692 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.687+0000 7f676cf4d640 1 -- 172.21.15.105:0/647520017 shutdown_connections 2024-09-21T01:46:32.692 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.687+0000 7f676cf4d640 1 --2- 172.21.15.105:0/647520017 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f67680721b0 0x7f67680980b0 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.692 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.687+0000 7f676cf4d640 1 --2- 172.21.15.105:0/647520017 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6768071860 0x7f6768071c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.692 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.687+0000 7f676cf4d640 1 -- 172.21.15.105:0/647520017 >> 172.21.15.105:0/647520017 conn(0x7f676809f000 msgr2=0x7f676809b4d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:32.692 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.687+0000 7f676cf4d640 1 -- 172.21.15.105:0/647520017 shutdown_connections 2024-09-21T01:46:32.692 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.687+0000 7f676cf4d640 1 -- 172.21.15.105:0/647520017 wait complete. 2024-09-21T01:46:32.693 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.688+0000 7ff9582a9640 1 Processor -- start 2024-09-21T01:46:32.693 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.688+0000 7ff9582a9640 1 -- start start 2024-09-21T01:46:32.693 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.688+0000 7fe47250d640 1 Processor -- start 2024-09-21T01:46:32.694 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7ff9582a9640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff950071860 0x7ff95012d950 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.694 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7ff9582a9640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff950072380 0x7ff95012de90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.694 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7f676cf4d640 1 Processor -- start 2024-09-21T01:46:32.694 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7f676cf4d640 1 -- start start 2024-09-21T01:46:32.694 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7fe47250d640 1 -- start start 2024-09-21T01:46:32.695 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7ff956045640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff950071860 0x7ff95012d950 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.695 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7f228942d640 1 Processor -- start 2024-09-21T01:46:32.695 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7ff9582a9640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff950131eb0 0x7ff95012e3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.696 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7ff956045640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff950071860 0x7ff95012d950 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:52100/0 (socket says 172.21.15.105:52100) 2024-09-21T01:46:32.696 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7ff956045640 1 -- 172.21.15.105:0/609605084 learned_addr learned my addr 172.21.15.105:0/609605084 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:32.696 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7ff9582a9640 1 -- 172.21.15.105:0/609605084 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7ff95006f620 con 0x7ff950071860 2024-09-21T01:46:32.696 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7ff9582a9640 1 -- 172.21.15.105:0/609605084 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7ff95006f4a0 con 0x7ff950131eb0 2024-09-21T01:46:32.696 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7ff9582a9640 1 -- 172.21.15.105:0/609605084 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7ff95006f7a0 con 0x7ff950072380 2024-09-21T01:46:32.696 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7f228942d640 1 -- start start 2024-09-21T01:46:32.697 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7ff955844640 1 --2- 172.21.15.105:0/609605084 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff950072380 0x7ff95012de90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.697 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7ff956846640 1 --2- 172.21.15.105:0/609605084 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff950131eb0 0x7ff95012e3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.697 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7ff956045640 1 -- 172.21.15.105:0/609605084 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff950072380 msgr2=0x7ff95012de90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.697 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7ff956045640 1 --2- 172.21.15.105:0/609605084 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff950072380 0x7ff95012de90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.697 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7ff956045640 1 -- 172.21.15.105:0/609605084 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff950131eb0 msgr2=0x7ff95012e3d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.698 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7ff956045640 1 --2- 172.21.15.105:0/609605084 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff950131eb0 0x7ff95012e3d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.698 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7ff956045640 1 -- 172.21.15.105:0/609605084 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff95012f860 con 0x7ff950071860 2024-09-21T01:46:32.698 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7ff956846640 1 --2- 172.21.15.105:0/609605084 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff950131eb0 0x7ff95012e3d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T01:46:32.698 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7ff955844640 1 --2- 172.21.15.105:0/609605084 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff950072380 0x7ff95012de90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-09-21T01:46:32.698 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.689+0000 7ff956045640 1 --2- 172.21.15.105:0/609605084 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff950071860 0x7ff95012d950 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7ff94c001a90 tx=0x7ff94c014420 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:32.698 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7ff946ffd640 1 -- 172.21.15.105:0/609605084 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff94c014b30 con 0x7ff950071860 2024-09-21T01:46:32.698 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7fe47250d640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fe46c070c50 0x7fe46c09b4b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.698 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7fe47250d640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fe46c071060 0x7fe46c09b9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.699 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7ff9582a9640 1 -- 172.21.15.105:0/609605084 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff95012c330 con 0x7ff950071860 2024-09-21T01:46:32.699 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7ff946ffd640 1 -- 172.21.15.105:0/609605084 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7ff94c014cd0 con 0x7ff950071860 2024-09-21T01:46:32.699 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7ff946ffd640 1 -- 172.21.15.105:0/609605084 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff94c005330 con 0x7ff950071860 2024-09-21T01:46:32.699 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7f676cf4d640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6768071860 0x7f6768091570 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.699 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7f676cf4d640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f67680721b0 0x7f6768091ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.700 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7ff9582a9640 1 -- 172.21.15.105:0/609605084 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff9501c1940 con 0x7ff950071860 2024-09-21T01:46:32.700 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7fe47250d640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe46c12f770 0x7fe46c09bf30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.700 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7f676cf4d640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f67680985f0 0x7f6768091ff0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.700 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7f228942d640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2284096b30 0x7f22840939c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.700 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7fe46b7fe640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fe46c071060 0x7fe46c09b9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.701 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7f6765d9b640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f67680721b0 0x7f6768091ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.701 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7f676cf4d640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f676806ed70 con 0x7f67680985f0 2024-09-21T01:46:32.701 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7f676cf4d640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f676806ebf0 con 0x7f6768071860 2024-09-21T01:46:32.701 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7f676cf4d640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f676806eef0 con 0x7f67680721b0 2024-09-21T01:46:32.701 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7f6765d9b640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f67680721b0 0x7f6768091ab0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:48336/0 (socket says 172.21.15.105:48336) 2024-09-21T01:46:32.701 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7f6765d9b640 1 -- 172.21.15.105:0/372527764 learned_addr learned my addr 172.21.15.105:0/372527764 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:32.701 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7f228942d640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2284097650 0x7f2284093f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.702 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7fe46bfff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fe46c070c50 0x7fe46c09b4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.702 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7fe470aaa640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe46c12f770 0x7fe46c09bf30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.702 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7fe46bfff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fe46c070c50 0x7fe46c09b4b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:50576/0 (socket says 172.21.15.105:50576) 2024-09-21T01:46:32.702 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7f228942d640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2284131e60 0x7f2284094440 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.703 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7f228942d640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f2284135300 con 0x7f2284097650 2024-09-21T01:46:32.703 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7f228942d640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f2284135180 con 0x7f2284096b30 2024-09-21T01:46:32.703 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7f228942d640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f2284135480 con 0x7f2284131e60 2024-09-21T01:46:32.703 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7f6766d9d640 1 --2- 172.21.15.105:0/372527764 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f67680985f0 0x7f6768091ff0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.703 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.690+0000 7fe46b7fe640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fe46c071060 0x7fe46c09b9f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:48340/0 (socket says 172.21.15.105:48340) 2024-09-21T01:46:32.704 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7f22837fe640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2284131e60 0x7f2284094440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.704 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7fe470aaa640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe46c12f770 0x7fe46c09bf30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:52116/0 (socket says 172.21.15.105:52116) 2024-09-21T01:46:32.704 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7fe47250d640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7fe46c1386e0 con 0x7fe46c12f770 2024-09-21T01:46:32.704 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7fe47250d640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7fe46c138560 con 0x7fe46c070c50 2024-09-21T01:46:32.704 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7fe47250d640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7fe46c138860 con 0x7fe46c071060 2024-09-21T01:46:32.704 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7fe46bfff640 1 -- 172.21.15.105:0/2465975097 learned_addr learned my addr 172.21.15.105:0/2465975097 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:32.705 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7f676659c640 1 --2- 172.21.15.105:0/372527764 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6768071860 0x7f6768091570 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.705 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7f6766d9d640 1 -- 172.21.15.105:0/372527764 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f67680721b0 msgr2=0x7f6768091ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.705 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7f6766d9d640 1 --2- 172.21.15.105:0/372527764 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f67680721b0 0x7f6768091ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.705 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7f6766d9d640 1 -- 172.21.15.105:0/372527764 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6768071860 msgr2=0x7f6768091570 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.705 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7f6766d9d640 1 --2- 172.21.15.105:0/372527764 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6768071860 0x7f6768091570 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.705 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7f6766d9d640 1 -- 172.21.15.105:0/372527764 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f67680936b0 con 0x7f67680985f0 2024-09-21T01:46:32.705 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7f22827fc640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2284097650 0x7f2284093f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.706 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7f22827fc640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2284097650 0x7f2284093f00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:52132/0 (socket says 172.21.15.105:52132) 2024-09-21T01:46:32.706 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7f22837fe640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2284131e60 0x7f2284094440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:48350/0 (socket says 172.21.15.105:48350) 2024-09-21T01:46:32.706 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7f22827fc640 1 -- 172.21.15.105:0/1933679494 learned_addr learned my addr 172.21.15.105:0/1933679494 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:32.706 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7f2282ffd640 1 --2- 172.21.15.105:0/1933679494 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2284096b30 0x7f22840939c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.706 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7f6766d9d640 1 --2- 172.21.15.105:0/372527764 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f67680985f0 0x7f6768091ff0 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f676000c700 tx=0x7f676000cbc0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:32.706 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7ff9582a9640 1 -- 172.21.15.105:0/609605084 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7ff950128d50 con 0x7ff950071860 2024-09-21T01:46:32.707 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7fe470aaa640 1 -- 172.21.15.105:0/2465975097 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fe46c071060 msgr2=0x7fe46c09b9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.707 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7fe470aaa640 1 --2- 172.21.15.105:0/2465975097 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fe46c071060 0x7fe46c09b9f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.707 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7fe470aaa640 1 -- 172.21.15.105:0/2465975097 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fe46c070c50 msgr2=0x7fe46c09b4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.707 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7fe470aaa640 1 --2- 172.21.15.105:0/2465975097 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fe46c070c50 0x7fe46c09b4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.707 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.692+0000 7ff946ffd640 1 -- 172.21.15.105:0/609605084 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7ff94c0120d0 con 0x7ff950071860 2024-09-21T01:46:32.707 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7fe470aaa640 1 -- 172.21.15.105:0/2465975097 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fe46c134710 con 0x7fe46c12f770 2024-09-21T01:46:32.708 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.692+0000 7f6756ffd640 1 -- 172.21.15.105:0/372527764 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6760003b70 con 0x7f67680985f0 2024-09-21T01:46:32.708 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.692+0000 7f6756ffd640 1 -- 172.21.15.105:0/372527764 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f6760003d10 con 0x7f67680985f0 2024-09-21T01:46:32.708 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.692+0000 7f6756ffd640 1 -- 172.21.15.105:0/372527764 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6760005310 con 0x7f67680985f0 2024-09-21T01:46:32.708 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.691+0000 7f22827fc640 1 -- 172.21.15.105:0/1933679494 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2284131e60 msgr2=0x7f2284094440 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.708 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.692+0000 7f22827fc640 1 --2- 172.21.15.105:0/1933679494 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f2284131e60 0x7f2284094440 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.709 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.692+0000 7f22827fc640 1 -- 172.21.15.105:0/1933679494 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2284096b30 msgr2=0x7f22840939c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.709 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.692+0000 7f22827fc640 1 --2- 172.21.15.105:0/1933679494 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f2284096b30 0x7f22840939c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.709 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.692+0000 7f22827fc640 1 -- 172.21.15.105:0/1933679494 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f22840962d0 con 0x7f2284097650 2024-09-21T01:46:32.709 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.692+0000 7f676cf4d640 1 -- 172.21.15.105:0/372527764 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6768094560 con 0x7f67680985f0 2024-09-21T01:46:32.709 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.692+0000 7fe470aaa640 1 --2- 172.21.15.105:0/2465975097 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe46c12f770 0x7fe46c09bf30 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7fe46400ec50 tx=0x7fe46400c6a0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:32.709 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.692+0000 7f22827fc640 1 --2- 172.21.15.105:0/1933679494 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2284097650 0x7f2284093f00 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f227800d980 tx=0x7f227800de40 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:32.710 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.692+0000 7f676cf4d640 1 -- 172.21.15.105:0/372527764 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f67681c5790 con 0x7f67680985f0 2024-09-21T01:46:32.710 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.692+0000 7ff946ffd640 1 -- 172.21.15.105:0/609605084 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7ff94c0050d0 con 0x7ff950071860 2024-09-21T01:46:32.710 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.692+0000 7ff946ffd640 1 -- 172.21.15.105:0/609605084 --> v1:172.21.15.105:6808/2933198036 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7ff93405bcf0 con 0x7ff934057ff0 2024-09-21T01:46:32.710 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.692+0000 7ff946ffd640 1 -- 172.21.15.105:0/609605084 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7ff94c0752a0 con 0x7ff950071860 2024-09-21T01:46:32.711 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.692+0000 7fe468ff9640 1 -- 172.21.15.105:0/2465975097 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe464003b70 con 0x7fe46c12f770 2024-09-21T01:46:32.711 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.692+0000 7fe468ff9640 1 -- 172.21.15.105:0/2465975097 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fe464003d10 con 0x7fe46c12f770 2024-09-21T01:46:32.711 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.693+0000 7fe47250d640 1 -- 172.21.15.105:0/2465975097 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fe46c1338b0 con 0x7fe46c12f770 2024-09-21T01:46:32.711 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.693+0000 7fe47250d640 1 -- 172.21.15.105:0/2465975097 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fe46c1319f0 con 0x7fe46c12f770 2024-09-21T01:46:32.711 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.693+0000 7fe468ff9640 1 -- 172.21.15.105:0/2465975097 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fe464004280 con 0x7fe46c12f770 2024-09-21T01:46:32.711 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.693+0000 7f6754ff9640 1 -- 172.21.15.105:0/372527764 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f6740001030 con 0x7f67680985f0 2024-09-21T01:46:32.712 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.693+0000 7f226b7fe640 1 -- 172.21.15.105:0/1933679494 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2278017070 con 0x7f2284097650 2024-09-21T01:46:32.712 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.693+0000 7f228942d640 1 -- 172.21.15.105:0/1933679494 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f2284095c30 con 0x7f2284097650 2024-09-21T01:46:32.712 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.693+0000 7ff946ffd640 1 -- 172.21.15.105:0/609605084 <== osd.2 v1:172.21.15.105:6808/2933198036 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7ff93405bcf0 con 0x7ff934057ff0 2024-09-21T01:46:32.712 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.694+0000 7fe47250d640 1 -- 172.21.15.105:0/2465975097 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7fe46c071730 con 0x7fe46c12f770 2024-09-21T01:46:32.713 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.693+0000 7f228942d640 1 -- 172.21.15.105:0/1933679494 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f22841c5960 con 0x7f2284097650 2024-09-21T01:46:32.713 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.694+0000 7f226b7fe640 1 -- 172.21.15.105:0/1933679494 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2278009c30 con 0x7f2284097650 2024-09-21T01:46:32.713 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.694+0000 7f6756ffd640 1 -- 172.21.15.105:0/372527764 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f67600054b0 con 0x7f67680985f0 2024-09-21T01:46:32.713 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.695+0000 7f6756ffd640 1 -- 172.21.15.105:0/372527764 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f6760068c60 con 0x7f67680985f0 2024-09-21T01:46:32.714 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.695+0000 7f226b7fe640 1 -- 172.21.15.105:0/1933679494 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2278020060 con 0x7f2284097650 2024-09-21T01:46:32.714 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.695+0000 7f228942d640 1 -- 172.21.15.105:0/1933679494 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f228408db60 con 0x7f2284097650 2024-09-21T01:46:32.714 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.695+0000 7fe468ff9640 1 -- 172.21.15.105:0/2465975097 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fe464004420 con 0x7fe46c12f770 2024-09-21T01:46:32.714 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.695+0000 7f6756ffd640 1 -- 172.21.15.105:0/372527764 --> v1:172.21.15.105:6804/1896987326 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f674805bb40 con 0x7f6748057e40 2024-09-21T01:46:32.714 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.695+0000 7f226b7fe640 1 -- 172.21.15.105:0/1933679494 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f2278010460 con 0x7f2284097650 2024-09-21T01:46:32.715 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.695+0000 7f6756ffd640 1 -- 172.21.15.105:0/372527764 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7f676006d2a0 con 0x7f67680985f0 2024-09-21T01:46:32.715 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.696+0000 7fe468ff9640 1 -- 172.21.15.105:0/2465975097 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fe464069cb0 con 0x7fe46c12f770 2024-09-21T01:46:32.715 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.696+0000 7f226b7fe640 1 -- 172.21.15.105:0/1933679494 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f2278069850 con 0x7f2284097650 2024-09-21T01:46:32.716 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.696+0000 7f226b7fe640 1 -- 172.21.15.105:0/1933679494 --> v1:172.21.15.105:6812/568702146 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f2278004470 con 0x7f2270057df0 2024-09-21T01:46:32.716 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.696+0000 7fe468ff9640 1 -- 172.21.15.105:0/2465975097 --> v1:172.21.15.105:6800/249246859 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7fe464010e50 con 0x7fe458057df0 2024-09-21T01:46:32.716 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.696+0000 7f226b7fe640 1 -- 172.21.15.105:0/1933679494 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7f227806e2a0 con 0x7f2284097650 2024-09-21T01:46:32.716 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.696+0000 7fe468ff9640 1 -- 172.21.15.105:0/2465975097 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7fe46406c310 con 0x7fe46c12f770 2024-09-21T01:46:32.717 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.696+0000 7f6756ffd640 1 -- 172.21.15.105:0/372527764 <== osd.1 v1:172.21.15.105:6804/1896987326 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f674805bb40 con 0x7f6748057e40 2024-09-21T01:46:32.717 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.697+0000 7fe468ff9640 1 -- 172.21.15.105:0/2465975097 <== osd.0 v1:172.21.15.105:6800/249246859 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7fe464010e50 con 0x7fe458057df0 2024-09-21T01:46:32.717 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.698+0000 7f226b7fe640 1 -- 172.21.15.105:0/1933679494 <== osd.3 v1:172.21.15.105:6812/568702146 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f2278004470 con 0x7f2270057df0 2024-09-21T01:46:32.724 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.724+0000 7ff9582a9640 1 -- 172.21.15.105:0/609605084 --> v1:172.21.15.105:6808/2933198036 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7ff95012f0a0 con 0x7ff934057ff0 2024-09-21T01:46:32.724 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.724+0000 7ff946ffd640 1 -- 172.21.15.105:0/609605084 <== osd.2 v1:172.21.15.105:6808/2933198036 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (unknown 0 0 1318614166) 0x7ff95012f0a0 con 0x7ff934057ff0 2024-09-21T01:46:32.725 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.724+0000 7ff9582a9640 1 -- 172.21.15.105:0/609605084 >> v1:172.21.15.105:6808/2933198036 conn(0x7ff934057ff0 legacy=0x7ff93405a480 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.725 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.725+0000 7ff9582a9640 1 -- 172.21.15.105:0/609605084 >> v1:172.21.15.105:6816/2838005468 conn(0x7ff9340516b0 legacy=0x7ff934053bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.725 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.725+0000 7ff9582a9640 1 -- 172.21.15.105:0/609605084 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff950071860 msgr2=0x7ff95012d950 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.725 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.725+0000 7ff9582a9640 1 --2- 172.21.15.105:0/609605084 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff950071860 0x7ff95012d950 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7ff94c001a90 tx=0x7ff94c014420 comp rx=0 tx=0).stop 2024-09-21T01:46:32.725 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.725+0000 7ff956846640 1 -- 172.21.15.105:0/609605084 reap_dead start 2024-09-21T01:46:32.726 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.725+0000 7ff9582a9640 1 -- 172.21.15.105:0/609605084 shutdown_connections 2024-09-21T01:46:32.726 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.725+0000 7ff9582a9640 1 -- 172.21.15.105:0/609605084 >> 172.21.15.105:0/609605084 conn(0x7ff95009f000 msgr2=0x7ff95009aea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:32.726 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.725+0000 7ff9582a9640 1 -- 172.21.15.105:0/609605084 shutdown_connections 2024-09-21T01:46:32.726 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.725+0000 7ff9582a9640 1 -- 172.21.15.105:0/609605084 wait complete. 2024-09-21T01:46:32.726 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.726+0000 7f6754ff9640 1 -- 172.21.15.105:0/372527764 --> v1:172.21.15.105:6804/1896987326 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f6740002dd0 con 0x7f6748057e40 2024-09-21T01:46:32.726 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.726+0000 7f6756ffd640 1 -- 172.21.15.105:0/372527764 <== osd.1 v1:172.21.15.105:6804/1896987326 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (unknown 0 0 3324758384) 0x7f6740002dd0 con 0x7f6748057e40 2024-09-21T01:46:32.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.728+0000 7f6754ff9640 1 -- 172.21.15.105:0/372527764 >> v1:172.21.15.105:6804/1896987326 conn(0x7f6748057e40 legacy=0x7f674805a2d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.728+0000 7f6754ff9640 1 -- 172.21.15.105:0/372527764 >> v1:172.21.15.105:6816/2838005468 conn(0x7f6748051500 legacy=0x7f6748053a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.728+0000 7f6754ff9640 1 -- 172.21.15.105:0/372527764 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f67680985f0 msgr2=0x7f6768091ff0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.728+0000 7f6754ff9640 1 --2- 172.21.15.105:0/372527764 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f67680985f0 0x7f6768091ff0 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7f676000c700 tx=0x7f676000cbc0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.728+0000 7f6766d9d640 1 -- 172.21.15.105:0/372527764 reap_dead start 2024-09-21T01:46:32.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.728+0000 7f6754ff9640 1 -- 172.21.15.105:0/372527764 shutdown_connections 2024-09-21T01:46:32.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.728+0000 7f6754ff9640 1 -- 172.21.15.105:0/372527764 >> 172.21.15.105:0/372527764 conn(0x7f676809f000 msgr2=0x7f6768126d40 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:32.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.728+0000 7f6754ff9640 1 -- 172.21.15.105:0/372527764 shutdown_connections 2024-09-21T01:46:32.729 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.728+0000 7f6754ff9640 1 -- 172.21.15.105:0/372527764 wait complete. 2024-09-21T01:46:32.730 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.729+0000 7fe47250d640 1 -- 172.21.15.105:0/2465975097 --> v1:172.21.15.105:6800/249246859 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7fe46c131fb0 con 0x7fe458057df0 2024-09-21T01:46:32.730 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.729+0000 7f228942d640 1 -- 172.21.15.105:0/1933679494 --> v1:172.21.15.105:6812/568702146 -- command(tid 2: {"prefix": "flush_pg_stats"}) -- 0x7f2284131280 con 0x7f2270057df0 2024-09-21T01:46:32.730 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.729+0000 7fe468ff9640 1 -- 172.21.15.105:0/2465975097 <== osd.0 v1:172.21.15.105:6800/249246859 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (unknown 0 0 795413372) 0x7fe46c131fb0 con 0x7fe458057df0 2024-09-21T01:46:32.731 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.729+0000 7f226b7fe640 1 -- 172.21.15.105:0/1933679494 <== osd.3 v1:172.21.15.105:6812/568702146 2 ==== command_reply(tid 2: 0 ) ==== 8+0+12 (unknown 0 0 1318614166) 0x7f2284131280 con 0x7f2270057df0 2024-09-21T01:46:32.731 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.730+0000 7fe47250d640 1 -- 172.21.15.105:0/2465975097 >> v1:172.21.15.105:6800/249246859 conn(0x7fe458057df0 legacy=0x7fe45805a280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.732 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.730+0000 7f228942d640 1 -- 172.21.15.105:0/1933679494 >> v1:172.21.15.105:6812/568702146 conn(0x7f2270057df0 legacy=0x7f227005a280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.732 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.730+0000 7f228942d640 1 -- 172.21.15.105:0/1933679494 >> v1:172.21.15.105:6816/2838005468 conn(0x7f22700514b0 legacy=0x7f22700539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.732 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.730+0000 7fe47250d640 1 -- 172.21.15.105:0/2465975097 >> v1:172.21.15.105:6816/2838005468 conn(0x7fe4580514b0 legacy=0x7fe4580539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.732 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.730+0000 7fe47250d640 1 -- 172.21.15.105:0/2465975097 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe46c12f770 msgr2=0x7fe46c09bf30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.732 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.730+0000 7fe47250d640 1 --2- 172.21.15.105:0/2465975097 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fe46c12f770 0x7fe46c09bf30 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7fe46400ec50 tx=0x7fe46400c6a0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.732 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.730+0000 7f228942d640 1 -- 172.21.15.105:0/1933679494 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2284097650 msgr2=0x7f2284093f00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.733 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.730+0000 7f228942d640 1 --2- 172.21.15.105:0/1933679494 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f2284097650 0x7f2284093f00 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f227800d980 tx=0x7f227800de40 comp rx=0 tx=0).stop 2024-09-21T01:46:32.733 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.730+0000 7fe470aaa640 1 -- 172.21.15.105:0/2465975097 reap_dead start 2024-09-21T01:46:32.733 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.730+0000 7f22837fe640 1 -- 172.21.15.105:0/1933679494 reap_dead start 2024-09-21T01:46:32.733 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.730+0000 7f228942d640 1 -- 172.21.15.105:0/1933679494 shutdown_connections 2024-09-21T01:46:32.733 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.730+0000 7f228942d640 1 -- 172.21.15.105:0/1933679494 >> 172.21.15.105:0/1933679494 conn(0x7f228409e4e0 msgr2=0x7f228412f910 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:32.734 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.730+0000 7fe47250d640 1 -- 172.21.15.105:0/2465975097 shutdown_connections 2024-09-21T01:46:32.734 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.730+0000 7fe47250d640 1 -- 172.21.15.105:0/2465975097 >> 172.21.15.105:0/2465975097 conn(0x7fe46c0a2080 msgr2=0x7fe46c12c490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:32.734 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.730+0000 7f228942d640 1 -- 172.21.15.105:0/1933679494 shutdown_connections 2024-09-21T01:46:32.734 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.730+0000 7f228942d640 1 -- 172.21.15.105:0/1933679494 wait complete. 2024-09-21T01:46:32.735 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.730+0000 7fe47250d640 1 -- 172.21.15.105:0/2465975097 shutdown_connections 2024-09-21T01:46:32.735 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.730+0000 7fe47250d640 1 -- 172.21.15.105:0/2465975097 wait complete. 2024-09-21T01:46:32.740 INFO:teuthology.orchestra.run.smithi105.stdout:124554051587 2024-09-21T01:46:32.741 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T01:46:32.745 INFO:teuthology.orchestra.run.smithi105.stdout:115964116995 2024-09-21T01:46:32.745 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T01:46:32.748 INFO:teuthology.orchestra.run.smithi105.stdout:124554051587 2024-09-21T01:46:32.749 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T01:46:32.749 INFO:teuthology.orchestra.run.smithi105.stdout:111669149699 2024-09-21T01:46:32.750 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T01:46:32.858 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.857+0000 7fa97bd17640 1 -- 172.21.15.105:0/2371948972 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa97412a400 msgr2=0x7fa97412a810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.858 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.857+0000 7fa97bd17640 1 --2- 172.21.15.105:0/2371948972 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa97412a400 0x7fa97412a810 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7fa964009900 tx=0x7fa96402a530 comp rx=0 tx=0).stop 2024-09-21T01:46:32.858 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.857+0000 7fa97bd17640 1 -- 172.21.15.105:0/2371948972 shutdown_connections 2024-09-21T01:46:32.858 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.857+0000 7fa97bd17640 1 --2- 172.21.15.105:0/2371948972 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa97412a400 0x7fa97412a810 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.858 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.857+0000 7fa97bd17640 1 --2- 172.21.15.105:0/2371948972 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa974073960 0x7fa974129ec0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.858 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.857+0000 7fa97bd17640 1 -- 172.21.15.105:0/2371948972 >> 172.21.15.105:0/2371948972 conn(0x7fa97409fc30 msgr2=0x7fa9740a0040 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:32.858 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.857+0000 7fa97bd17640 1 -- 172.21.15.105:0/2371948972 shutdown_connections 2024-09-21T01:46:32.858 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.857+0000 7fa97bd17640 1 -- 172.21.15.105:0/2371948972 wait complete. 2024-09-21T01:46:32.859 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.859+0000 7fa97bd17640 1 Processor -- start 2024-09-21T01:46:32.859 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.859+0000 7fa97bd17640 1 -- start start 2024-09-21T01:46:32.862 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.860+0000 7fa97bd17640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa974073960 0x7fa9740990c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.862 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.860+0000 7fa97bd17640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa97412a400 0x7fa974099600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.862 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.860+0000 7fa979ab3640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa974073960 0x7fa9740990c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.862 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.860+0000 7fa979ab3640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa974073960 0x7fa9740990c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:48372/0 (socket says 172.21.15.105:48372) 2024-09-21T01:46:32.862 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.860+0000 7fa9792b2640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa97412a400 0x7fa974099600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.862 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.860+0000 7fa9792b2640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa97412a400 0x7fa974099600 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:50614/0 (socket says 172.21.15.105:50614) 2024-09-21T01:46:32.863 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.860+0000 7fa979ab3640 1 -- 172.21.15.105:0/4046198534 learned_addr learned my addr 172.21.15.105:0/4046198534 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:32.863 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.860+0000 7fa97bd17640 1 --2- 172.21.15.105:0/4046198534 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa97412d4a0 0x7fa974099b40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.863 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.860+0000 7fa97bd17640 1 -- 172.21.15.105:0/4046198534 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7fa9741363e0 con 0x7fa97412d4a0 2024-09-21T01:46:32.863 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.860+0000 7fa97bd17640 1 -- 172.21.15.105:0/4046198534 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7fa974136260 con 0x7fa97412a400 2024-09-21T01:46:32.863 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.860+0000 7fa97bd17640 1 -- 172.21.15.105:0/4046198534 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7fa974136560 con 0x7fa974073960 2024-09-21T01:46:32.863 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.860+0000 7fa97a2b4640 1 --2- 172.21.15.105:0/4046198534 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa97412d4a0 0x7fa974099b40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.863 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.860+0000 7fa979ab3640 1 -- 172.21.15.105:0/4046198534 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa97412a400 msgr2=0x7fa974099600 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.863 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.861+0000 7fa979ab3640 1 --2- 172.21.15.105:0/4046198534 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa97412a400 0x7fa974099600 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.863 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.861+0000 7fa979ab3640 1 -- 172.21.15.105:0/4046198534 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa97412d4a0 msgr2=0x7fa974099b40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.863 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.861+0000 7fa979ab3640 1 --2- 172.21.15.105:0/4046198534 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa97412d4a0 0x7fa974099b40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.863 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.861+0000 7fa979ab3640 1 -- 172.21.15.105:0/4046198534 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa974134140 con 0x7fa974073960 2024-09-21T01:46:32.863 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.861+0000 7fa97a2b4640 1 --2- 172.21.15.105:0/4046198534 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa97412d4a0 0x7fa974099b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:32.863 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.861+0000 7fa9792b2640 1 --2- 172.21.15.105:0/4046198534 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa97412a400 0x7fa974099600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:32.863 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.861+0000 7fa979ab3640 1 --2- 172.21.15.105:0/4046198534 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa974073960 0x7fa9740990c0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7fa97000e9f0 tx=0x7fa97000eeb0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:32.864 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.861+0000 7fa96a7fc640 1 -- 172.21.15.105:0/4046198534 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa97000cd80 con 0x7fa974073960 2024-09-21T01:46:32.864 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.861+0000 7fa96a7fc640 1 -- 172.21.15.105:0/4046198534 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fa970004020 con 0x7fa974073960 2024-09-21T01:46:32.864 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.861+0000 7fa96a7fc640 1 -- 172.21.15.105:0/4046198534 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa970004470 con 0x7fa974073960 2024-09-21T01:46:32.864 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.862+0000 7fa97bd17640 1 -- 172.21.15.105:0/4046198534 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa9741338d0 con 0x7fa974073960 2024-09-21T01:46:32.864 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.862+0000 7fa97bd17640 1 -- 172.21.15.105:0/4046198534 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fa974131840 con 0x7fa974073960 2024-09-21T01:46:32.864 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.864+0000 7fa97bd17640 1 -- 172.21.15.105:0/4046198534 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa9740a0040 con 0x7fa974073960 2024-09-21T01:46:32.864 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.864+0000 7fa96a7fc640 1 -- 172.21.15.105:0/4046198534 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fa9700035f0 con 0x7fa974073960 2024-09-21T01:46:32.865 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.865+0000 7fa96a7fc640 1 -- 172.21.15.105:0/4046198534 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fa970069910 con 0x7fa974073960 2024-09-21T01:46:32.872 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.872+0000 7fa96a7fc640 1 -- 172.21.15.105:0/4046198534 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fa9740a0040 con 0x7fa974073960 2024-09-21T01:46:32.910 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.909+0000 7f5e642f6640 1 -- 172.21.15.105:0/4225218963 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5e5c12cd70 msgr2=0x7f5e5c094780 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.910 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.909+0000 7f5e642f6640 1 --2- 172.21.15.105:0/4225218963 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5e5c12cd70 0x7f5e5c094780 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f5e54009be0 tx=0x7f5e5402a6f0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.910 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.909+0000 7f5e642f6640 1 -- 172.21.15.105:0/4225218963 shutdown_connections 2024-09-21T01:46:32.910 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.910+0000 7f5e642f6640 1 --2- 172.21.15.105:0/4225218963 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5e5c12cd70 0x7f5e5c094780 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.911 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.910+0000 7f5e642f6640 1 --2- 172.21.15.105:0/4225218963 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f5e5c06cd90 0x7f5e5c12c420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.911 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.910+0000 7f5e642f6640 1 -- 172.21.15.105:0/4225218963 >> 172.21.15.105:0/4225218963 conn(0x7f5e5c09e470 msgr2=0x7f5e5c09e880 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:32.911 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.910+0000 7f5e642f6640 1 -- 172.21.15.105:0/4225218963 shutdown_connections 2024-09-21T01:46:32.911 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.910+0000 7f5e642f6640 1 -- 172.21.15.105:0/4225218963 wait complete. 2024-09-21T01:46:32.911 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.911+0000 7f5e642f6640 1 Processor -- start 2024-09-21T01:46:32.911 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.911+0000 7f5e642f6640 1 -- start start 2024-09-21T01:46:32.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.911+0000 7f5e642f6640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5e5c12c960 0x7f5e5c130f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.911+0000 7f5e642f6640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f5e5c12cd70 0x7f5e5c1314a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.911+0000 7f5e642f6640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5e5c133760 0x7f5e5c1319e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.911+0000 7f5e642f6640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f5e5c1352c0 con 0x7f5e5c12c960 2024-09-21T01:46:32.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.912+0000 7f5e642f6640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f5e5c135140 con 0x7f5e5c12cd70 2024-09-21T01:46:32.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.912+0000 7f5e642f6640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f5e5c135440 con 0x7f5e5c133760 2024-09-21T01:46:32.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.912+0000 7f5e62092640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5e5c12c960 0x7f5e5c130f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.912+0000 7f5e62092640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5e5c12c960 0x7f5e5c130f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:52154/0 (socket says 172.21.15.105:52154) 2024-09-21T01:46:32.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.912+0000 7f5e62092640 1 -- 172.21.15.105:0/1798166663 learned_addr learned my addr 172.21.15.105:0/1798166663 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:32.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.912+0000 7f5e61891640 1 --2- 172.21.15.105:0/1798166663 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f5e5c12cd70 0x7f5e5c1314a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.912+0000 7f5e62893640 1 --2- 172.21.15.105:0/1798166663 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5e5c133760 0x7f5e5c1319e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.914 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.912+0000 7f5e62092640 1 -- 172.21.15.105:0/1798166663 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5e5c133760 msgr2=0x7f5e5c1319e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.915 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.912+0000 7f5e62092640 1 --2- 172.21.15.105:0/1798166663 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5e5c133760 0x7f5e5c1319e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.915 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.912+0000 7f5e62092640 1 -- 172.21.15.105:0/1798166663 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f5e5c12cd70 msgr2=0x7f5e5c1314a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.915 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.912+0000 7f5e62092640 1 --2- 172.21.15.105:0/1798166663 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f5e5c12cd70 0x7f5e5c1314a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.915 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.912+0000 7f5e62092640 1 -- 172.21.15.105:0/1798166663 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5e5c1330d0 con 0x7f5e5c12c960 2024-09-21T01:46:32.915 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.912+0000 7f5e61891640 1 --2- 172.21.15.105:0/1798166663 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f5e5c12cd70 0x7f5e5c1314a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:32.915 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.913+0000 7f5e62092640 1 --2- 172.21.15.105:0/1798166663 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5e5c12c960 0x7f5e5c130f60 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f5e5800e9f0 tx=0x7f5e5800eeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:32.915 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.913+0000 7f5e52ffd640 1 -- 172.21.15.105:0/1798166663 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5e5800cd80 con 0x7f5e5c12c960 2024-09-21T01:46:32.915 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.913+0000 7f5e642f6640 1 -- 172.21.15.105:0/1798166663 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5e5c133e90 con 0x7f5e5c12c960 2024-09-21T01:46:32.915 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.913+0000 7f5e52ffd640 1 -- 172.21.15.105:0/1798166663 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f5e58004020 con 0x7f5e5c12c960 2024-09-21T01:46:32.915 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.913+0000 7f5e52ffd640 1 -- 172.21.15.105:0/1798166663 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5e58004410 con 0x7f5e5c12c960 2024-09-21T01:46:32.915 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.913+0000 7f5e642f6640 1 -- 172.21.15.105:0/1798166663 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f5e5c135b60 con 0x7f5e5c12c960 2024-09-21T01:46:32.916 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.914+0000 7fd8b1499640 1 -- 172.21.15.105:0/4037570268 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd8ac12fd00 msgr2=0x7fd8ac137990 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.916 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.914+0000 7fd8b1499640 1 --2- 172.21.15.105:0/4037570268 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd8ac12fd00 0x7fd8ac137990 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7fd8a4005fd0 tx=0x7fd8a402a820 comp rx=0 tx=0).stop 2024-09-21T01:46:32.916 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.914+0000 7fd8b1499640 1 -- 172.21.15.105:0/4037570268 shutdown_connections 2024-09-21T01:46:32.916 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.914+0000 7fd8b1499640 1 --2- 172.21.15.105:0/4037570268 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd8ac12fd00 0x7fd8ac137990 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.916 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.914+0000 7fd8b1499640 1 --2- 172.21.15.105:0/4037570268 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd8ac12c200 0x7fd8ac12c610 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.916 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.914+0000 7fd8b1499640 1 -- 172.21.15.105:0/4037570268 >> 172.21.15.105:0/4037570268 conn(0x7fd8ac0a1f40 msgr2=0x7fd8ac0a2350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:32.916 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.914+0000 7fd8b1499640 1 -- 172.21.15.105:0/4037570268 shutdown_connections 2024-09-21T01:46:32.917 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.914+0000 7f5e642f6640 1 -- 172.21.15.105:0/1798166663 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5e5c06cd90 con 0x7f5e5c12c960 2024-09-21T01:46:32.917 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.914+0000 7fd8b1499640 1 -- 172.21.15.105:0/4037570268 wait complete. 2024-09-21T01:46:32.917 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.915+0000 7f5e52ffd640 1 -- 172.21.15.105:0/1798166663 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f5e58005020 con 0x7f5e5c12c960 2024-09-21T01:46:32.917 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.915+0000 7f5e52ffd640 1 -- 172.21.15.105:0/1798166663 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f5e580699d0 con 0x7f5e5c12c960 2024-09-21T01:46:32.919 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.916+0000 7fd8b1499640 1 Processor -- start 2024-09-21T01:46:32.919 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.916+0000 7fd8b1499640 1 -- start start 2024-09-21T01:46:32.920 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.917+0000 7fd8b1499640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd8ac12c200 0x7fd8ac09b4b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.920 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.917+0000 7fd8b1499640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd8ac12cb50 0x7fd8ac09b9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.920 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.917+0000 7fd8aaffd640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd8ac12c200 0x7fd8ac09b4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.920 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.917+0000 7fd8aaffd640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd8ac12c200 0x7fd8ac09b4b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:48408/0 (socket says 172.21.15.105:48408) 2024-09-21T01:46:32.920 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.917+0000 7fd8b1499640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd8ac12fd00 0x7fd8ac09bf30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.920 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.917+0000 7fd8aaffd640 1 -- 172.21.15.105:0/461291018 learned_addr learned my addr 172.21.15.105:0/461291018 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:32.920 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.917+0000 7fd8b1499640 1 -- 172.21.15.105:0/461291018 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7fd8ac1386d0 con 0x7fd8ac12cb50 2024-09-21T01:46:32.920 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.917+0000 7fd8b1499640 1 -- 172.21.15.105:0/461291018 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7fd8ac138550 con 0x7fd8ac12fd00 2024-09-21T01:46:32.920 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.917+0000 7fd8b1499640 1 -- 172.21.15.105:0/461291018 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7fd8ac138850 con 0x7fd8ac12c200 2024-09-21T01:46:32.920 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.917+0000 7fd8ab7fe640 1 --2- 172.21.15.105:0/461291018 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd8ac12fd00 0x7fd8ac09bf30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.920 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.917+0000 7fd8aa7fc640 1 --2- 172.21.15.105:0/461291018 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd8ac12cb50 0x7fd8ac09b9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.920 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.918+0000 7fd8aaffd640 1 -- 172.21.15.105:0/461291018 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd8ac12fd00 msgr2=0x7fd8ac09bf30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.920 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.918+0000 7fd8aaffd640 1 --2- 172.21.15.105:0/461291018 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd8ac12fd00 0x7fd8ac09bf30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.920 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.918+0000 7fd8aaffd640 1 -- 172.21.15.105:0/461291018 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd8ac12cb50 msgr2=0x7fd8ac09b9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.921 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.918+0000 7fd8aaffd640 1 --2- 172.21.15.105:0/461291018 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd8ac12cb50 0x7fd8ac09b9f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.921 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.918+0000 7fd8aaffd640 1 -- 172.21.15.105:0/461291018 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fd8ac134980 con 0x7fd8ac12c200 2024-09-21T01:46:32.921 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.918+0000 7fd8ab7fe640 1 --2- 172.21.15.105:0/461291018 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd8ac12fd00 0x7fd8ac09bf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:32.921 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.918+0000 7fd8aa7fc640 1 --2- 172.21.15.105:0/461291018 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd8ac12cb50 0x7fd8ac09b9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:32.921 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.918+0000 7fd8aaffd640 1 --2- 172.21.15.105:0/461291018 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd8ac12c200 0x7fd8ac09b4b0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7fd89c00ea60 tx=0x7fd89c00ef20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:32.921 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.918+0000 7fd8937fe640 1 -- 172.21.15.105:0/461291018 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd89c00cde0 con 0x7fd8ac12c200 2024-09-21T01:46:32.921 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.918+0000 7fd8937fe640 1 -- 172.21.15.105:0/461291018 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fd89c004020 con 0x7fd8ac12c200 2024-09-21T01:46:32.921 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.918+0000 7fd8937fe640 1 -- 172.21.15.105:0/461291018 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fd89c004450 con 0x7fd8ac12c200 2024-09-21T01:46:32.921 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.919+0000 7fd8b1499640 1 -- 172.21.15.105:0/461291018 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fd8ac134460 con 0x7fd8ac12c200 2024-09-21T01:46:32.921 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.919+0000 7fd8b1499640 1 -- 172.21.15.105:0/461291018 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7fd8ac131ce0 con 0x7fd8ac12c200 2024-09-21T01:46:32.922 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.920+0000 7fd8b1499640 1 -- 172.21.15.105:0/461291018 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fd8ac12a600 con 0x7fd8ac12c200 2024-09-21T01:46:32.922 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.920+0000 7fd8937fe640 1 -- 172.21.15.105:0/461291018 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fd89c0035f0 con 0x7fd8ac12c200 2024-09-21T01:46:32.922 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.921+0000 7fd8937fe640 1 -- 172.21.15.105:0/461291018 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fd89c069970 con 0x7fd8ac12c200 2024-09-21T01:46:32.922 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.922+0000 7f5e52ffd640 1 -- 172.21.15.105:0/1798166663 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f5e58009490 con 0x7f5e5c12c960 2024-09-21T01:46:32.923 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.923+0000 7feedd551640 1 -- 172.21.15.105:0/4105811583 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feed812c8e0 msgr2=0x7feed812ccf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.923 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.923+0000 7feedd551640 1 --2- 172.21.15.105:0/4105811583 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feed812c8e0 0x7feed812ccf0 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7feec8009830 tx=0x7feec802a4b0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.924 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.923+0000 7feedd551640 1 -- 172.21.15.105:0/4105811583 shutdown_connections 2024-09-21T01:46:32.924 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.923+0000 7feedd551640 1 --2- 172.21.15.105:0/4105811583 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feed8097670 0x7feed8134730 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.924 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.923+0000 7feedd551640 1 --2- 172.21.15.105:0/4105811583 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feed812c8e0 0x7feed812ccf0 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.924 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.923+0000 7feedd551640 1 -- 172.21.15.105:0/4105811583 >> 172.21.15.105:0/4105811583 conn(0x7feed809e4e0 msgr2=0x7feed809e8f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:32.924 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.923+0000 7feedd551640 1 -- 172.21.15.105:0/4105811583 shutdown_connections 2024-09-21T01:46:32.924 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.923+0000 7feedd551640 1 -- 172.21.15.105:0/4105811583 wait complete. 2024-09-21T01:46:32.924 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.924+0000 7feedd551640 1 Processor -- start 2024-09-21T01:46:32.924 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.924+0000 7feedd551640 1 -- start start 2024-09-21T01:46:32.925 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.925+0000 7feedd551640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feed8096b50 0x7feed81c3b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.925+0000 7feedd551640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7feed8097670 0x7feed81c40c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.925+0000 7feedd551640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feed812c8e0 0x7feed81c4600 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:32.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.925+0000 7feedd551640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7feed81355b0 con 0x7feed812c8e0 2024-09-21T01:46:32.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.925+0000 7feedd551640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7feed8135430 con 0x7feed8096b50 2024-09-21T01:46:32.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.925+0000 7feedd551640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7feed8135730 con 0x7feed8097670 2024-09-21T01:46:32.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.925+0000 7feed6ffd640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feed8096b50 0x7feed81c3b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.926 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.925+0000 7fd8937fe640 1 -- 172.21.15.105:0/461291018 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fd8ac12a600 con 0x7fd8ac12c200 2024-09-21T01:46:32.928 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.925+0000 7feed6ffd640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feed8096b50 0x7feed81c3b80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:50668/0 (socket says 172.21.15.105:50668) 2024-09-21T01:46:32.928 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.925+0000 7feed6ffd640 1 -- 172.21.15.105:0/2051124461 learned_addr learned my addr 172.21.15.105:0/2051124461 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:32.928 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.925+0000 7feed77fe640 1 --2- 172.21.15.105:0/2051124461 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feed812c8e0 0x7feed81c4600 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.928 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.925+0000 7feed67fc640 1 --2- 172.21.15.105:0/2051124461 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7feed8097670 0x7feed81c40c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:32.928 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.926+0000 7feed77fe640 1 -- 172.21.15.105:0/2051124461 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7feed8097670 msgr2=0x7feed81c40c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.928 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.926+0000 7feed77fe640 1 --2- 172.21.15.105:0/2051124461 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7feed8097670 0x7feed81c40c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.928 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.926+0000 7feed77fe640 1 -- 172.21.15.105:0/2051124461 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feed8096b50 msgr2=0x7feed81c3b80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:32.929 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.926+0000 7feed77fe640 1 --2- 172.21.15.105:0/2051124461 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feed8096b50 0x7feed81c3b80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:32.929 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.926+0000 7feed77fe640 1 -- 172.21.15.105:0/2051124461 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7feed808ff90 con 0x7feed812c8e0 2024-09-21T01:46:32.929 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.926+0000 7feed6ffd640 1 --2- 172.21.15.105:0/2051124461 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feed8096b50 0x7feed81c3b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:32.929 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.926+0000 7feed77fe640 1 --2- 172.21.15.105:0/2051124461 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feed812c8e0 0x7feed81c4600 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7feed0009fd0 tx=0x7feed000ef10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:32.929 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.926+0000 7feebf7fe640 1 -- 172.21.15.105:0/2051124461 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feed000cdc0 con 0x7feed812c8e0 2024-09-21T01:46:32.929 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.926+0000 7feebf7fe640 1 -- 172.21.15.105:0/2051124461 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7feed0004c10 con 0x7feed812c8e0 2024-09-21T01:46:32.929 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.927+0000 7feedd551640 1 -- 172.21.15.105:0/2051124461 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7feed808f760 con 0x7feed812c8e0 2024-09-21T01:46:32.929 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.927+0000 7feedd551640 1 -- 172.21.15.105:0/2051124461 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7feed812b3e0 con 0x7feed812c8e0 2024-09-21T01:46:32.930 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.928+0000 7feebf7fe640 1 -- 172.21.15.105:0/2051124461 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7feed0005020 con 0x7feed812c8e0 2024-09-21T01:46:32.930 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.928+0000 7feebf7fe640 1 -- 172.21.15.105:0/2051124461 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7feed0005240 con 0x7feed812c8e0 2024-09-21T01:46:32.930 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.929+0000 7feedd551640 1 -- 172.21.15.105:0/2051124461 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7feed812ccf0 con 0x7feed812c8e0 2024-09-21T01:46:32.930 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.929+0000 7feebf7fe640 1 -- 172.21.15.105:0/2051124461 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7feed0069bf0 con 0x7feed812c8e0 2024-09-21T01:46:32.937 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:32.937+0000 7feebf7fe640 1 -- 172.21.15.105:0/2051124461 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7feed00688c0 con 0x7feed812c8e0 2024-09-21T01:46:33.049 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.049+0000 7fa97bd17640 1 -- 172.21.15.105:0/4046198534 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7fa974129ec0 con 0x7fa974073960 2024-09-21T01:46:33.049 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.049+0000 7fa96a7fc640 1 -- 172.21.15.105:0/4046198534 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+2 (secure 0 0 0) 0x7fa970009490 con 0x7fa974073960 2024-09-21T01:46:33.050 INFO:teuthology.orchestra.run.smithi105.stdout:1 2024-09-21T01:46:33.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.051+0000 7fa97bd17640 1 -- 172.21.15.105:0/4046198534 >> v1:172.21.15.105:6816/2838005468 conn(0x7fa9580558c0 legacy=0x7fa958057dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:33.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.051+0000 7fa97bd17640 1 -- 172.21.15.105:0/4046198534 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa974073960 msgr2=0x7fa9740990c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:33.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.051+0000 7fa97bd17640 1 --2- 172.21.15.105:0/4046198534 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa974073960 0x7fa9740990c0 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7fa97000e9f0 tx=0x7fa97000eeb0 comp rx=0 tx=0).stop 2024-09-21T01:46:33.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.051+0000 7fa97bd17640 1 -- 172.21.15.105:0/4046198534 shutdown_connections 2024-09-21T01:46:33.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.051+0000 7fa97bd17640 1 --2- 172.21.15.105:0/4046198534 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa97412d4a0 0x7fa974099b40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:33.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.051+0000 7fa97bd17640 1 --2- 172.21.15.105:0/4046198534 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa97412a400 0x7fa974099600 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:33.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.051+0000 7fa97bd17640 1 --2- 172.21.15.105:0/4046198534 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa974073960 0x7fa9740990c0 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:33.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.051+0000 7fa97bd17640 1 -- 172.21.15.105:0/4046198534 >> 172.21.15.105:0/4046198534 conn(0x7fa97409fc30 msgr2=0x7fa974127130 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:33.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.051+0000 7fa97bd17640 1 -- 172.21.15.105:0/4046198534 shutdown_connections 2024-09-21T01:46:33.052 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.051+0000 7fa97bd17640 1 -- 172.21.15.105:0/4046198534 wait complete. 2024-09-21T01:46:33.063 INFO:tasks.ceph.ceph_manager.ceph:need seq 124554051587 got 1 for osd.2 2024-09-21T01:46:33.083 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.083+0000 7f5e642f6640 1 -- 172.21.15.105:0/1798166663 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7f5e5c0736c0 con 0x7f5e5c12c960 2024-09-21T01:46:33.084 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.083+0000 7f5e52ffd640 1 -- 172.21.15.105:0/1798166663 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f5e58068960 con 0x7f5e5c12c960 2024-09-21T01:46:33.084 INFO:teuthology.orchestra.run.smithi105.stdout:111669149698 2024-09-21T01:46:33.085 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.085+0000 7fd8b1499640 1 -- 172.21.15.105:0/461291018 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7fd8ac0a26d0 con 0x7fd8ac12c200 2024-09-21T01:46:33.085 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.085+0000 7f5e642f6640 1 -- 172.21.15.105:0/1798166663 >> v1:172.21.15.105:6816/2838005468 conn(0x7f5e400514b0 legacy=0x7f5e400539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:33.086 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.085+0000 7f5e642f6640 1 -- 172.21.15.105:0/1798166663 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5e5c12c960 msgr2=0x7f5e5c130f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:33.086 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.085+0000 7f5e642f6640 1 --2- 172.21.15.105:0/1798166663 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5e5c12c960 0x7f5e5c130f60 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f5e5800e9f0 tx=0x7f5e5800eeb0 comp rx=0 tx=0).stop 2024-09-21T01:46:33.086 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.085+0000 7f5e642f6640 1 -- 172.21.15.105:0/1798166663 shutdown_connections 2024-09-21T01:46:33.086 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.085+0000 7f5e642f6640 1 --2- 172.21.15.105:0/1798166663 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5e5c133760 0x7f5e5c1319e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:33.086 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.085+0000 7f5e642f6640 1 --2- 172.21.15.105:0/1798166663 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f5e5c12cd70 0x7f5e5c1314a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:33.086 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.085+0000 7f5e642f6640 1 --2- 172.21.15.105:0/1798166663 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5e5c12c960 0x7f5e5c130f60 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:33.086 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.085+0000 7fd8937fe640 1 -- 172.21.15.105:0/461291018 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7fd8ac0a26d0 con 0x7fd8ac12c200 2024-09-21T01:46:33.087 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.085+0000 7f5e642f6640 1 -- 172.21.15.105:0/1798166663 >> 172.21.15.105:0/1798166663 conn(0x7f5e5c09e470 msgr2=0x7f5e5c070890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:33.087 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.085+0000 7f5e642f6640 1 -- 172.21.15.105:0/1798166663 shutdown_connections 2024-09-21T01:46:33.087 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.085+0000 7f5e642f6640 1 -- 172.21.15.105:0/1798166663 wait complete. 2024-09-21T01:46:33.087 INFO:teuthology.orchestra.run.smithi105.stdout:115964116994 2024-09-21T01:46:33.088 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.087+0000 7fd8b1499640 1 -- 172.21.15.105:0/461291018 >> v1:172.21.15.105:6816/2838005468 conn(0x7fd8980514b0 legacy=0x7fd8980539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:33.088 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.087+0000 7fd8b1499640 1 -- 172.21.15.105:0/461291018 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd8ac12c200 msgr2=0x7fd8ac09b4b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:33.088 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.087+0000 7fd8b1499640 1 --2- 172.21.15.105:0/461291018 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd8ac12c200 0x7fd8ac09b4b0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7fd89c00ea60 tx=0x7fd89c00ef20 comp rx=0 tx=0).stop 2024-09-21T01:46:33.088 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.087+0000 7fd8b1499640 1 -- 172.21.15.105:0/461291018 shutdown_connections 2024-09-21T01:46:33.088 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.087+0000 7fd8b1499640 1 --2- 172.21.15.105:0/461291018 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fd8ac12fd00 0x7fd8ac09bf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:33.088 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.087+0000 7fd8b1499640 1 --2- 172.21.15.105:0/461291018 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fd8ac12cb50 0x7fd8ac09b9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:33.088 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.087+0000 7fd8b1499640 1 --2- 172.21.15.105:0/461291018 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fd8ac12c200 0x7fd8ac09b4b0 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:33.088 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.087+0000 7fd8b1499640 1 -- 172.21.15.105:0/461291018 >> 172.21.15.105:0/461291018 conn(0x7fd8ac0a1f40 msgr2=0x7fd8ac12f400 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:33.088 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.087+0000 7fd8b1499640 1 -- 172.21.15.105:0/461291018 shutdown_connections 2024-09-21T01:46:33.088 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.087+0000 7fd8b1499640 1 -- 172.21.15.105:0/461291018 wait complete. 2024-09-21T01:46:33.094 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.094+0000 7feedd551640 1 -- 172.21.15.105:0/2051124461 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7feed8075190 con 0x7feed812c8e0 2024-09-21T01:46:33.095 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.095+0000 7feebf7fe640 1 -- 172.21.15.105:0/2051124461 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+2 (secure 0 0 0) 0x7feed00429f0 con 0x7feed812c8e0 2024-09-21T01:46:33.095 INFO:teuthology.orchestra.run.smithi105.stdout:1 2024-09-21T01:46:33.097 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.097+0000 7feedd551640 1 -- 172.21.15.105:0/2051124461 >> v1:172.21.15.105:6816/2838005468 conn(0x7feec4051500 legacy=0x7feec4053a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:33.097 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.097+0000 7feedd551640 1 -- 172.21.15.105:0/2051124461 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feed812c8e0 msgr2=0x7feed81c4600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:33.097 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.097+0000 7feedd551640 1 --2- 172.21.15.105:0/2051124461 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feed812c8e0 0x7feed81c4600 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7feed0009fd0 tx=0x7feed000ef10 comp rx=0 tx=0).stop 2024-09-21T01:46:33.098 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.097+0000 7feedd551640 1 -- 172.21.15.105:0/2051124461 shutdown_connections 2024-09-21T01:46:33.098 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.097+0000 7feedd551640 1 --2- 172.21.15.105:0/2051124461 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7feed812c8e0 0x7feed81c4600 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:33.098 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.097+0000 7feedd551640 1 --2- 172.21.15.105:0/2051124461 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7feed8097670 0x7feed81c40c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:33.098 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.097+0000 7feedd551640 1 --2- 172.21.15.105:0/2051124461 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7feed8096b50 0x7feed81c3b80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:33.098 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.097+0000 7feedd551640 1 -- 172.21.15.105:0/2051124461 >> 172.21.15.105:0/2051124461 conn(0x7feed809e4e0 msgr2=0x7feed80741f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:33.098 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.097+0000 7feedd551640 1 -- 172.21.15.105:0/2051124461 shutdown_connections 2024-09-21T01:46:33.098 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:33.097+0000 7feedd551640 1 -- 172.21.15.105:0/2051124461 wait complete. 2024-09-21T01:46:33.099 INFO:tasks.ceph.ceph_manager.ceph:need seq 111669149699 got 111669149698 for osd.0 2024-09-21T01:46:33.100 INFO:tasks.ceph.ceph_manager.ceph:need seq 115964116995 got 115964116994 for osd.1 2024-09-21T01:46:33.109 INFO:tasks.ceph.ceph_manager.ceph:need seq 124554051587 got 1 for osd.3 2024-09-21T01:46:34.065 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.2 2024-09-21T01:46:34.100 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.0 2024-09-21T01:46:34.101 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.1 2024-09-21T01:46:34.109 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd last-stat-seq osd.3 2024-09-21T01:46:34.197 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.196+0000 7f51d99c7640 1 -- 172.21.15.105:0/2539100592 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f51d40976e0 msgr2=0x7f51d4097af0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.197 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.196+0000 7f51d99c7640 1 --2- 172.21.15.105:0/2539100592 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f51d40976e0 0x7f51d4097af0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f51c400d930 tx=0x7f51c402c410 comp rx=0 tx=0).stop 2024-09-21T01:46:34.197 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.196+0000 7f51d99c7640 1 -- 172.21.15.105:0/2539100592 shutdown_connections 2024-09-21T01:46:34.197 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.196+0000 7f51d99c7640 1 --2- 172.21.15.105:0/2539100592 >> v2:172.21.15.105:3300/0 conn(0x7f51d4098030 0x7f51d4094070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.197 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.196+0000 7f51d99c7640 1 --2- 172.21.15.105:0/2539100592 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f51d40976e0 0x7f51d4097af0 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.197 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.196+0000 7f51d99c7640 1 -- 172.21.15.105:0/2539100592 >> 172.21.15.105:0/2539100592 conn(0x7f51d409e6f0 msgr2=0x7f51d409abc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:34.198 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.197+0000 7f51d99c7640 1 -- 172.21.15.105:0/2539100592 shutdown_connections 2024-09-21T01:46:34.198 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.197+0000 7f51d99c7640 1 -- 172.21.15.105:0/2539100592 wait complete. 2024-09-21T01:46:34.198 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.198+0000 7f51d99c7640 1 Processor -- start 2024-09-21T01:46:34.198 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.198+0000 7f51d99c7640 1 -- start start 2024-09-21T01:46:34.199 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.199+0000 7f51d99c7640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f51d40945b0 0x7f51d4128080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:34.199 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.199+0000 7f51d99c7640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f51d40976e0 0x7f51d41285c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:34.199 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.199+0000 7f51d99c7640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f51d4098030 0x7f51d4128b00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:34.200 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.199+0000 7f51d99c7640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f51d4096f00 con 0x7f51d4098030 2024-09-21T01:46:34.201 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.199+0000 7f51d99c7640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f51d4096d80 con 0x7f51d40945b0 2024-09-21T01:46:34.201 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.199+0000 7f51d99c7640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f51d4097080 con 0x7f51d40976e0 2024-09-21T01:46:34.201 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.199+0000 7f51d2ffd640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f51d40945b0 0x7f51d4128080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:34.201 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.199+0000 7f51d37fe640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f51d4098030 0x7f51d4128b00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:34.201 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.199+0000 7f51d2ffd640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f51d40945b0 0x7f51d4128080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:50690/0 (socket says 172.21.15.105:50690) 2024-09-21T01:46:34.201 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.199+0000 7f51d2ffd640 1 -- 172.21.15.105:0/2990476140 learned_addr learned my addr 172.21.15.105:0/2990476140 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:34.201 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.199+0000 7f51d27fc640 1 --2- 172.21.15.105:0/2990476140 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f51d40976e0 0x7f51d41285c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:34.201 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.199+0000 7f51d2ffd640 1 -- 172.21.15.105:0/2990476140 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f51d40976e0 msgr2=0x7f51d41285c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.201 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.199+0000 7f51d2ffd640 1 --2- 172.21.15.105:0/2990476140 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f51d40976e0 0x7f51d41285c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.201 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.199+0000 7f51d2ffd640 1 -- 172.21.15.105:0/2990476140 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f51d4098030 msgr2=0x7f51d4128b00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.201 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.199+0000 7f51d2ffd640 1 --2- 172.21.15.105:0/2990476140 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f51d4098030 0x7f51d4128b00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.201 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.199+0000 7f51d2ffd640 1 -- 172.21.15.105:0/2990476140 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f51d408ced0 con 0x7f51d40945b0 2024-09-21T01:46:34.201 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.199+0000 7f51d27fc640 1 --2- 172.21.15.105:0/2990476140 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f51d40976e0 0x7f51d41285c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:34.202 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.200+0000 7f51d2ffd640 1 --2- 172.21.15.105:0/2990476140 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f51d40945b0 0x7f51d4128080 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f51c400da50 tx=0x7f51c4032650 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:34.202 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.200+0000 7f51bbfff640 1 -- 172.21.15.105:0/2990476140 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f51c4005c90 con 0x7f51d40945b0 2024-09-21T01:46:34.202 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.200+0000 7f51bbfff640 1 -- 172.21.15.105:0/2990476140 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f51c4042070 con 0x7f51d40945b0 2024-09-21T01:46:34.202 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.200+0000 7f51d99c7640 1 -- 172.21.15.105:0/2990476140 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f51d408c9b0 con 0x7f51d40945b0 2024-09-21T01:46:34.202 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.200+0000 7f51bbfff640 1 -- 172.21.15.105:0/2990476140 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f51c4032aa0 con 0x7f51d40945b0 2024-09-21T01:46:34.202 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.200+0000 7f51d99c7640 1 -- 172.21.15.105:0/2990476140 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f51d41c5d40 con 0x7f51d40945b0 2024-09-21T01:46:34.202 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.201+0000 7f51d99c7640 1 -- 172.21.15.105:0/2990476140 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f51d4097af0 con 0x7f51d40945b0 2024-09-21T01:46:34.203 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.202+0000 7f51bbfff640 1 -- 172.21.15.105:0/2990476140 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f51c403b040 con 0x7f51d40945b0 2024-09-21T01:46:34.203 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.202+0000 7f51bbfff640 1 -- 172.21.15.105:0/2990476140 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f51c408a3a0 con 0x7f51d40945b0 2024-09-21T01:46:34.206 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.206+0000 7f51bbfff640 1 -- 172.21.15.105:0/2990476140 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f51c4033770 con 0x7f51d40945b0 2024-09-21T01:46:34.235 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.234+0000 7fb6967ae640 1 -- 172.21.15.105:0/130790284 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb69012c200 msgr2=0x7fb69012c610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.235 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.234+0000 7fb6967ae640 1 --2- 172.21.15.105:0/130790284 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb69012c200 0x7fb69012c610 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7fb680009830 tx=0x7fb68002a480 comp rx=0 tx=0).stop 2024-09-21T01:46:34.235 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.234+0000 7fb6967ae640 1 -- 172.21.15.105:0/130790284 shutdown_connections 2024-09-21T01:46:34.235 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.234+0000 7fb6967ae640 1 --2- 172.21.15.105:0/130790284 >> v2:172.21.15.105:3300/0 conn(0x7fb69012fd00 0x7fb690137990 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.235 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.234+0000 7fb6967ae640 1 --2- 172.21.15.105:0/130790284 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb69012c200 0x7fb69012c610 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.235 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.234+0000 7fb6967ae640 1 -- 172.21.15.105:0/130790284 >> 172.21.15.105:0/130790284 conn(0x7fb6900a1f40 msgr2=0x7fb6900a2350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:34.235 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.234+0000 7fb6967ae640 1 -- 172.21.15.105:0/130790284 shutdown_connections 2024-09-21T01:46:34.235 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.234+0000 7fb6967ae640 1 -- 172.21.15.105:0/130790284 wait complete. 2024-09-21T01:46:34.238 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.235+0000 7f6fbafcb640 1 -- 172.21.15.105:0/2466033573 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6fb40721b0 msgr2=0x7f6fb40980b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.238 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.235+0000 7f6fbafcb640 1 --2- 172.21.15.105:0/2466033573 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6fb40721b0 0x7f6fb40980b0 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7f6fa0009900 tx=0x7f6fa002a5b0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.238 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.235+0000 7f6fbafcb640 1 -- 172.21.15.105:0/2466033573 shutdown_connections 2024-09-21T01:46:34.238 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.235+0000 7f6fbafcb640 1 --2- 172.21.15.105:0/2466033573 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6fb40721b0 0x7f6fb40980b0 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.238 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.235+0000 7f6fbafcb640 1 --2- 172.21.15.105:0/2466033573 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6fb4071860 0x7f6fb4071c70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.238 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.235+0000 7f6fbafcb640 1 -- 172.21.15.105:0/2466033573 >> 172.21.15.105:0/2466033573 conn(0x7f6fb409f000 msgr2=0x7f6fb409b4d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:34.238 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.235+0000 7f6fbafcb640 1 -- 172.21.15.105:0/2466033573 shutdown_connections 2024-09-21T01:46:34.238 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.235+0000 7f6fbafcb640 1 -- 172.21.15.105:0/2466033573 wait complete. 2024-09-21T01:46:34.238 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.236+0000 7fb6967ae640 1 Processor -- start 2024-09-21T01:46:34.239 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.236+0000 7f6fbafcb640 1 Processor -- start 2024-09-21T01:46:34.239 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.236+0000 7f6fbafcb640 1 -- start start 2024-09-21T01:46:34.239 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.237+0000 7fb6967ae640 1 -- start start 2024-09-21T01:46:34.240 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.237+0000 7f6fbafcb640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6fb4071860 0x7f6fb41c39d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:34.240 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.237+0000 7f6fbafcb640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6fb40721b0 0x7f6fb41c3f10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:34.241 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.237+0000 7f6fbafcb640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6fb40985f0 0x7f6fb41c4450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:34.241 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.237+0000 7f6fbafcb640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f6fb4135570 con 0x7f6fb40985f0 2024-09-21T01:46:34.241 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.237+0000 7f6fbafcb640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f6fb41353f0 con 0x7f6fb4071860 2024-09-21T01:46:34.241 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.237+0000 7f6fbafcb640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f6fb41356f0 con 0x7f6fb40721b0 2024-09-21T01:46:34.241 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.237+0000 7f6fb3fff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6fb40721b0 0x7f6fb41c3f10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:34.241 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.237+0000 7f6fb3fff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6fb40721b0 0x7f6fb41c3f10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:48448/0 (socket says 172.21.15.105:48448) 2024-09-21T01:46:34.241 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.237+0000 7f6fb3fff640 1 -- 172.21.15.105:0/2279258356 learned_addr learned my addr 172.21.15.105:0/2279258356 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:34.241 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.237+0000 7f6fb9568640 1 --2- 172.21.15.105:0/2279258356 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6fb40985f0 0x7f6fb41c4450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:34.241 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.237+0000 7f6fb8d67640 1 --2- 172.21.15.105:0/2279258356 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6fb4071860 0x7f6fb41c39d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:34.241 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.238+0000 7f6fb3fff640 1 -- 172.21.15.105:0/2279258356 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6fb4071860 msgr2=0x7f6fb41c39d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.241 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.238+0000 7f6fb3fff640 1 --2- 172.21.15.105:0/2279258356 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6fb4071860 0x7f6fb41c39d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.241 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.238+0000 7f6fb3fff640 1 -- 172.21.15.105:0/2279258356 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6fb40985f0 msgr2=0x7f6fb41c4450 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.241 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.238+0000 7f6fb3fff640 1 --2- 172.21.15.105:0/2279258356 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6fb40985f0 0x7f6fb41c4450 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.241 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.238+0000 7f6fb3fff640 1 -- 172.21.15.105:0/2279258356 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6fb4132db0 con 0x7f6fb40721b0 2024-09-21T01:46:34.242 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.238+0000 7f6fb9568640 1 --2- 172.21.15.105:0/2279258356 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6fb40985f0 0x7f6fb41c4450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:34.242 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.238+0000 7fb6967ae640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb69012c200 0x7fb69009b4b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:34.242 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.238+0000 7fb6967ae640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb69012cb50 0x7fb69009b9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:34.242 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.238+0000 7f6fb3fff640 1 --2- 172.21.15.105:0/2279258356 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6fb40721b0 0x7f6fb41c3f10 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f6fa0000c00 tx=0x7f6fa00038a0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:34.243 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.238+0000 7f6fb17fa640 1 -- 172.21.15.105:0/2279258356 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6fa003f070 con 0x7f6fb40721b0 2024-09-21T01:46:34.243 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.238+0000 7fb68f7fe640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb69012cb50 0x7fb69009b9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:34.243 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.238+0000 7fb68ffff640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb69012c200 0x7fb69009b4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:34.243 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.238+0000 7fb68f7fe640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb69012cb50 0x7fb69009b9f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:50736/0 (socket says 172.21.15.105:50736) 2024-09-21T01:46:34.243 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.238+0000 7fb68ffff640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb69012c200 0x7fb69009b4b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:52206/0 (socket says 172.21.15.105:52206) 2024-09-21T01:46:34.243 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.238+0000 7fb6967ae640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb69012fd00 0x7fb69009bf30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:34.244 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.238+0000 7f6fb17fa640 1 -- 172.21.15.105:0/2279258356 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f6fa00047a0 con 0x7f6fb40721b0 2024-09-21T01:46:34.244 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.238+0000 7f6fbafcb640 1 -- 172.21.15.105:0/2279258356 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6fb41339d0 con 0x7f6fb40721b0 2024-09-21T01:46:34.244 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.238+0000 7f6fbafcb640 1 -- 172.21.15.105:0/2279258356 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f6fb4133eb0 con 0x7f6fb40721b0 2024-09-21T01:46:34.244 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.238+0000 7f6fb17fa640 1 -- 172.21.15.105:0/2279258356 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6fa0035030 con 0x7f6fb40721b0 2024-09-21T01:46:34.245 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.238+0000 7fb68f7fe640 1 -- 172.21.15.105:0/3992023314 learned_addr learned my addr 172.21.15.105:0/3992023314 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:34.245 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.238+0000 7fb6967ae640 1 -- 172.21.15.105:0/3992023314 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7fb690138720 con 0x7fb69012c200 2024-09-21T01:46:34.245 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.239+0000 7fb6967ae640 1 -- 172.21.15.105:0/3992023314 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7fb6901385a0 con 0x7fb69012cb50 2024-09-21T01:46:34.245 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.239+0000 7fb6967ae640 1 -- 172.21.15.105:0/3992023314 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7fb6901388a0 con 0x7fb69012fd00 2024-09-21T01:46:34.245 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.239+0000 7fb68f7fe640 1 -- 172.21.15.105:0/3992023314 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb69012fd00 msgr2=0x7fb69009bf30 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-09-21T01:46:34.245 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.239+0000 7fb694d4b640 1 --2- 172.21.15.105:0/3992023314 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb69012fd00 0x7fb69009bf30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:34.245 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.239+0000 7fb68f7fe640 1 --2- 172.21.15.105:0/3992023314 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb69012fd00 0x7fb69009bf30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.245 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.239+0000 7fb68f7fe640 1 -- 172.21.15.105:0/3992023314 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb69012c200 msgr2=0x7fb69009b4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.246 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.239+0000 7fb68f7fe640 1 --2- 172.21.15.105:0/3992023314 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb69012c200 0x7fb69009b4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.246 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.239+0000 7fb68f7fe640 1 -- 172.21.15.105:0/3992023314 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb6901346d0 con 0x7fb69012cb50 2024-09-21T01:46:34.246 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.239+0000 7fb68ffff640 1 --2- 172.21.15.105:0/3992023314 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb69012c200 0x7fb69009b4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T01:46:34.246 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.239+0000 7fb694d4b640 1 --2- 172.21.15.105:0/3992023314 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb69012fd00 0x7fb69009bf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-09-21T01:46:34.246 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.240+0000 7fb68f7fe640 1 --2- 172.21.15.105:0/3992023314 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb69012cb50 0x7fb69009b9f0 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7fb68400c450 tx=0x7fb68400c910 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:34.246 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.240+0000 7fb68cff9640 1 -- 172.21.15.105:0/3992023314 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb684016020 con 0x7fb69012cb50 2024-09-21T01:46:34.246 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.240+0000 7fb68cff9640 1 -- 172.21.15.105:0/3992023314 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fb684012920 con 0x7fb69012cb50 2024-09-21T01:46:34.246 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.240+0000 7fb68cff9640 1 -- 172.21.15.105:0/3992023314 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb684012d50 con 0x7fb69012cb50 2024-09-21T01:46:34.246 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.240+0000 7fb6967ae640 1 -- 172.21.15.105:0/3992023314 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb690134080 con 0x7fb69012cb50 2024-09-21T01:46:34.246 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.240+0000 7f6fbafcb640 1 -- 172.21.15.105:0/2279258356 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f6fb4131760 con 0x7f6fb40721b0 2024-09-21T01:46:34.246 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.240+0000 7f6fb17fa640 1 -- 172.21.15.105:0/2279258356 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f6fa0042070 con 0x7f6fb40721b0 2024-09-21T01:46:34.247 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.240+0000 7fb6967ae640 1 -- 172.21.15.105:0/3992023314 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7fb6901316d0 con 0x7fb69012cb50 2024-09-21T01:46:34.247 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.241+0000 7f6fb17fa640 1 -- 172.21.15.105:0/2279258356 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f6fa008a2b0 con 0x7f6fb40721b0 2024-09-21T01:46:34.247 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.242+0000 7fb68cff9640 1 -- 172.21.15.105:0/3992023314 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fb684019020 con 0x7fb69012cb50 2024-09-21T01:46:34.248 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.242+0000 7fb6967ae640 1 -- 172.21.15.105:0/3992023314 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb69012c610 con 0x7fb69012cb50 2024-09-21T01:46:34.248 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.243+0000 7fb68cff9640 1 -- 172.21.15.105:0/3992023314 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fb684068d10 con 0x7fb69012cb50 2024-09-21T01:46:34.248 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.245+0000 7f6fb17fa640 1 -- 172.21.15.105:0/2279258356 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f6fb4131760 con 0x7f6fb40721b0 2024-09-21T01:46:34.248 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.248+0000 7fb68cff9640 1 -- 172.21.15.105:0/3992023314 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fb68406e020 con 0x7fb69012cb50 2024-09-21T01:46:34.276 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.275+0000 7fa0dbdc4640 1 -- 172.21.15.105:0/1474663611 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa0d4071730 msgr2=0x7fa0d4134940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.277 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.275+0000 7fa0dbdc4640 1 --2- 172.21.15.105:0/1474663611 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa0d4071730 0x7fa0d4134940 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7fa0c4009830 tx=0x7fa0c402a480 comp rx=0 tx=0).stop 2024-09-21T01:46:34.277 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.275+0000 7fa0dbdc4640 1 -- 172.21.15.105:0/1474663611 shutdown_connections 2024-09-21T01:46:34.277 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.275+0000 7fa0dbdc4640 1 --2- 172.21.15.105:0/1474663611 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa0d4134e80 0x7fa0d41372a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.277 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.275+0000 7fa0dbdc4640 1 --2- 172.21.15.105:0/1474663611 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa0d4071730 0x7fa0d4134940 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.277 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.275+0000 7fa0dbdc4640 1 -- 172.21.15.105:0/1474663611 >> 172.21.15.105:0/1474663611 conn(0x7fa0d40a2080 msgr2=0x7fa0d40a2490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:34.277 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.276+0000 7fa0dbdc4640 1 -- 172.21.15.105:0/1474663611 shutdown_connections 2024-09-21T01:46:34.277 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.276+0000 7fa0dbdc4640 1 -- 172.21.15.105:0/1474663611 wait complete. 2024-09-21T01:46:34.279 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.277+0000 7fa0dbdc4640 1 Processor -- start 2024-09-21T01:46:34.279 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.277+0000 7fa0dbdc4640 1 -- start start 2024-09-21T01:46:34.279 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.277+0000 7fa0dbdc4640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa0d4071730 0x7fa0d409b4b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:34.279 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.277+0000 7fa0dbdc4640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa0d412f770 0x7fa0d409b9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:34.279 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.277+0000 7fa0dbdc4640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa0d4134e80 0x7fa0d409bf30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:34.279 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.277+0000 7fa0dbdc4640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7fa0d41386a0 con 0x7fa0d4134e80 2024-09-21T01:46:34.280 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.277+0000 7fa0d935f640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa0d412f770 0x7fa0d409b9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:34.280 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.277+0000 7fa0d9b60640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa0d4071730 0x7fa0d409b4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:34.280 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.277+0000 7fa0d935f640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa0d412f770 0x7fa0d409b9f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:48472/0 (socket says 172.21.15.105:48472) 2024-09-21T01:46:34.280 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.277+0000 7fa0d935f640 1 -- 172.21.15.105:0/3557676752 learned_addr learned my addr 172.21.15.105:0/3557676752 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:34.280 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.278+0000 7fa0da361640 1 --2- 172.21.15.105:0/3557676752 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa0d4134e80 0x7fa0d409bf30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:34.280 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.278+0000 7fa0dbdc4640 1 -- 172.21.15.105:0/3557676752 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7fa0d4138520 con 0x7fa0d4071730 2024-09-21T01:46:34.280 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.278+0000 7fa0dbdc4640 1 -- 172.21.15.105:0/3557676752 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7fa0d4138820 con 0x7fa0d412f770 2024-09-21T01:46:34.280 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.278+0000 7fa0da361640 1 -- 172.21.15.105:0/3557676752 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa0d412f770 msgr2=0x7fa0d409b9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.280 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.278+0000 7fa0da361640 1 --2- 172.21.15.105:0/3557676752 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa0d412f770 0x7fa0d409b9f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.280 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.278+0000 7fa0da361640 1 -- 172.21.15.105:0/3557676752 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa0d4071730 msgr2=0x7fa0d409b4b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.280 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.278+0000 7fa0da361640 1 --2- 172.21.15.105:0/3557676752 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa0d4071730 0x7fa0d409b4b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.280 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.278+0000 7fa0da361640 1 -- 172.21.15.105:0/3557676752 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fa0d41345e0 con 0x7fa0d4134e80 2024-09-21T01:46:34.280 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.278+0000 7fa0d935f640 1 --2- 172.21.15.105:0/3557676752 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa0d412f770 0x7fa0d409b9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:34.280 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.278+0000 7fa0d9b60640 1 --2- 172.21.15.105:0/3557676752 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa0d4071730 0x7fa0d409b4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:34.281 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.278+0000 7fa0da361640 1 --2- 172.21.15.105:0/3557676752 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa0d4134e80 0x7fa0d409bf30 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7fa0cc00e9e0 tx=0x7fa0cc00eea0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:34.281 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.278+0000 7fa0ca7fc640 1 -- 172.21.15.105:0/3557676752 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa0cc00cdb0 con 0x7fa0d4134e80 2024-09-21T01:46:34.281 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.279+0000 7fa0ca7fc640 1 -- 172.21.15.105:0/3557676752 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fa0cc004b20 con 0x7fa0d4134e80 2024-09-21T01:46:34.281 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.279+0000 7fa0dbdc4640 1 -- 172.21.15.105:0/3557676752 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fa0d4131060 con 0x7fa0d4134e80 2024-09-21T01:46:34.281 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.279+0000 7fa0ca7fc640 1 -- 172.21.15.105:0/3557676752 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fa0cc005020 con 0x7fa0d4134e80 2024-09-21T01:46:34.281 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.279+0000 7fa0dbdc4640 1 -- 172.21.15.105:0/3557676752 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fa0d409d890 con 0x7fa0d4134e80 2024-09-21T01:46:34.281 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.280+0000 7fa0dbdc4640 1 -- 172.21.15.105:0/3557676752 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fa0d4070c50 con 0x7fa0d4134e80 2024-09-21T01:46:34.282 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.280+0000 7fa0ca7fc640 1 -- 172.21.15.105:0/3557676752 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fa0cc0051c0 con 0x7fa0d4134e80 2024-09-21T01:46:34.282 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.281+0000 7fa0ca7fc640 1 -- 172.21.15.105:0/3557676752 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fa0cc069cf0 con 0x7fa0d4134e80 2024-09-21T01:46:34.285 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.285+0000 7fa0ca7fc640 1 -- 172.21.15.105:0/3557676752 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fa0cc068d80 con 0x7fa0d4134e80 2024-09-21T01:46:34.404 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.403+0000 7f6fbafcb640 1 -- 172.21.15.105:0/2279258356 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) -- 0x7f6fb4131d00 con 0x7f6fb40721b0 2024-09-21T01:46:34.404 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.403+0000 7f6fb17fa640 1 -- 172.21.15.105:0/2279258356 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f6fa0089240 con 0x7f6fb40721b0 2024-09-21T01:46:34.404 INFO:teuthology.orchestra.run.smithi105.stdout:115964116995 2024-09-21T01:46:34.405 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.404+0000 7f51d99c7640 1 -- 172.21.15.105:0/2990476140 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) -- 0x7f51d406a820 con 0x7f51d40945b0 2024-09-21T01:46:34.405 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.404+0000 7f51bbfff640 1 -- 172.21.15.105:0/2990476140 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7f51c40893b0 con 0x7f51d40945b0 2024-09-21T01:46:34.405 INFO:teuthology.orchestra.run.smithi105.stdout:124554051587 2024-09-21T01:46:34.407 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.406+0000 7f6fbafcb640 1 -- 172.21.15.105:0/2279258356 >> v1:172.21.15.105:6816/2838005468 conn(0x7f6f980514c0 legacy=0x7f6f980539d0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.407 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.406+0000 7f6fbafcb640 1 -- 172.21.15.105:0/2279258356 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6fb40721b0 msgr2=0x7f6fb41c3f10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.407 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.406+0000 7f6fbafcb640 1 --2- 172.21.15.105:0/2279258356 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6fb40721b0 0x7f6fb41c3f10 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f6fa0000c00 tx=0x7f6fa00038a0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.407 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.406+0000 7f6fbafcb640 1 -- 172.21.15.105:0/2279258356 shutdown_connections 2024-09-21T01:46:34.407 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.406+0000 7f6fbafcb640 1 --2- 172.21.15.105:0/2279258356 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6fb40985f0 0x7f6fb41c4450 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.408 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.406+0000 7f6fbafcb640 1 --2- 172.21.15.105:0/2279258356 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6fb40721b0 0x7f6fb41c3f10 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.408 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.406+0000 7f6fbafcb640 1 --2- 172.21.15.105:0/2279258356 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6fb4071860 0x7f6fb41c39d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.408 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.406+0000 7f6fbafcb640 1 -- 172.21.15.105:0/2279258356 >> 172.21.15.105:0/2279258356 conn(0x7f6fb409f000 msgr2=0x7f6fb4125290 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:34.408 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.407+0000 7f6fbafcb640 1 -- 172.21.15.105:0/2279258356 shutdown_connections 2024-09-21T01:46:34.408 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.407+0000 7f6fbafcb640 1 -- 172.21.15.105:0/2279258356 wait complete. 2024-09-21T01:46:34.409 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.407+0000 7f51d99c7640 1 -- 172.21.15.105:0/2990476140 >> v1:172.21.15.105:6816/2838005468 conn(0x7f51c00514b0 legacy=0x7f51c00539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.409 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.407+0000 7f51d99c7640 1 -- 172.21.15.105:0/2990476140 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f51d40945b0 msgr2=0x7f51d4128080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.409 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.407+0000 7f51d99c7640 1 --2- 172.21.15.105:0/2990476140 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f51d40945b0 0x7f51d4128080 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f51c400da50 tx=0x7f51c4032650 comp rx=0 tx=0).stop 2024-09-21T01:46:34.409 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.407+0000 7f51d99c7640 1 -- 172.21.15.105:0/2990476140 shutdown_connections 2024-09-21T01:46:34.409 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.407+0000 7f51d99c7640 1 --2- 172.21.15.105:0/2990476140 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f51d4098030 0x7f51d4128b00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.409 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.407+0000 7f51d99c7640 1 --2- 172.21.15.105:0/2990476140 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f51d40976e0 0x7f51d41285c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.409 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.407+0000 7f51d99c7640 1 --2- 172.21.15.105:0/2990476140 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f51d40945b0 0x7f51d4128080 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.410 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.407+0000 7f51d99c7640 1 -- 172.21.15.105:0/2990476140 >> 172.21.15.105:0/2990476140 conn(0x7f51d409e6f0 msgr2=0x7f51d412b430 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:34.410 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.407+0000 7f51d99c7640 1 -- 172.21.15.105:0/2990476140 shutdown_connections 2024-09-21T01:46:34.410 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.407+0000 7f51d99c7640 1 -- 172.21.15.105:0/2990476140 wait complete. 2024-09-21T01:46:34.420 INFO:tasks.ceph.ceph_manager.ceph:need seq 115964116995 got 115964116995 for osd.1 2024-09-21T01:46:34.420 DEBUG:teuthology.parallel:result is None 2024-09-21T01:46:34.421 INFO:tasks.ceph.ceph_manager.ceph:need seq 124554051587 got 124554051587 for osd.2 2024-09-21T01:46:34.421 DEBUG:teuthology.parallel:result is None 2024-09-21T01:46:34.442 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.441+0000 7fa0dbdc4640 1 -- 172.21.15.105:0/3557676752 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) -- 0x7fa0d406a780 con 0x7fa0d4134e80 2024-09-21T01:46:34.442 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.441+0000 7fa0ca7fc640 1 -- 172.21.15.105:0/3557676752 <== mon.0 v2:172.21.15.105:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7fa0cc0688c0 con 0x7fa0d4134e80 2024-09-21T01:46:34.442 INFO:teuthology.orchestra.run.smithi105.stdout:124554051587 2024-09-21T01:46:34.443 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.442+0000 7fb6967ae640 1 -- 172.21.15.105:0/3992023314 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) -- 0x7fb69006a820 con 0x7fb69012cb50 2024-09-21T01:46:34.443 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.443+0000 7fb68cff9640 1 -- 172.21.15.105:0/3992023314 <== mon.1 v2:172.21.15.105:3301/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) ==== 74+0+13 (secure 0 0 0) 0x7fb684020030 con 0x7fb69012cb50 2024-09-21T01:46:34.443 INFO:teuthology.orchestra.run.smithi105.stdout:111669149699 2024-09-21T01:46:34.444 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.443+0000 7fa0dbdc4640 1 -- 172.21.15.105:0/3557676752 >> v1:172.21.15.105:6816/2838005468 conn(0x7fa0b80514b0 legacy=0x7fa0b80539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.444 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.443+0000 7fa0dbdc4640 1 -- 172.21.15.105:0/3557676752 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa0d4134e80 msgr2=0x7fa0d409bf30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.444 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.443+0000 7fa0dbdc4640 1 --2- 172.21.15.105:0/3557676752 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa0d4134e80 0x7fa0d409bf30 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7fa0cc00e9e0 tx=0x7fa0cc00eea0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.444 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.443+0000 7fa0dbdc4640 1 -- 172.21.15.105:0/3557676752 shutdown_connections 2024-09-21T01:46:34.445 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.443+0000 7fa0dbdc4640 1 --2- 172.21.15.105:0/3557676752 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fa0d4134e80 0x7fa0d409bf30 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.445 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.443+0000 7fa0dbdc4640 1 --2- 172.21.15.105:0/3557676752 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fa0d412f770 0x7fa0d409b9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.445 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.443+0000 7fa0dbdc4640 1 --2- 172.21.15.105:0/3557676752 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fa0d4071730 0x7fa0d409b4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.445 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.443+0000 7fa0dbdc4640 1 -- 172.21.15.105:0/3557676752 >> 172.21.15.105:0/3557676752 conn(0x7fa0d40a2080 msgr2=0x7fa0d4073560 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:34.445 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.444+0000 7fa0dbdc4640 1 -- 172.21.15.105:0/3557676752 shutdown_connections 2024-09-21T01:46:34.445 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.444+0000 7fa0dbdc4640 1 -- 172.21.15.105:0/3557676752 wait complete. 2024-09-21T01:46:34.446 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.445+0000 7fb6967ae640 1 -- 172.21.15.105:0/3992023314 >> v1:172.21.15.105:6816/2838005468 conn(0x7fb67c0514b0 legacy=0x7fb67c0539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.446 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.445+0000 7fb6967ae640 1 -- 172.21.15.105:0/3992023314 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb69012cb50 msgr2=0x7fb69009b9f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.446 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.445+0000 7fb6967ae640 1 --2- 172.21.15.105:0/3992023314 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb69012cb50 0x7fb69009b9f0 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7fb68400c450 tx=0x7fb68400c910 comp rx=0 tx=0).stop 2024-09-21T01:46:34.446 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.445+0000 7fb6967ae640 1 -- 172.21.15.105:0/3992023314 shutdown_connections 2024-09-21T01:46:34.446 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.445+0000 7fb6967ae640 1 --2- 172.21.15.105:0/3992023314 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb69012fd00 0x7fb69009bf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.447 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.445+0000 7fb6967ae640 1 --2- 172.21.15.105:0/3992023314 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb69012cb50 0x7fb69009b9f0 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.447 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.445+0000 7fb6967ae640 1 --2- 172.21.15.105:0/3992023314 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb69012c200 0x7fb69009b4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.447 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.445+0000 7fb6967ae640 1 -- 172.21.15.105:0/3992023314 >> 172.21.15.105:0/3992023314 conn(0x7fb6900a1f40 msgr2=0x7fb69012f400 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:34.447 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.445+0000 7fb6967ae640 1 -- 172.21.15.105:0/3992023314 shutdown_connections 2024-09-21T01:46:34.447 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.445+0000 7fb6967ae640 1 -- 172.21.15.105:0/3992023314 wait complete. 2024-09-21T01:46:34.456 INFO:tasks.ceph.ceph_manager.ceph:need seq 124554051587 got 124554051587 for osd.3 2024-09-21T01:46:34.456 DEBUG:teuthology.parallel:result is None 2024-09-21T01:46:34.457 INFO:tasks.ceph.ceph_manager.ceph:need seq 111669149699 got 111669149699 for osd.0 2024-09-21T01:46:34.457 DEBUG:teuthology.parallel:result is None 2024-09-21T01:46:34.457 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean 2024-09-21T01:46:34.457 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T01:46:34.609 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.608+0000 7f0c11f1b640 1 -- 172.21.15.105:0/3976473814 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0c0c0733e0 msgr2=0x7f0c0c0737f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.609 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.608+0000 7f0c11f1b640 1 --2- 172.21.15.105:0/3976473814 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0c0c0733e0 0x7f0c0c0737f0 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f0bfc009860 tx=0x7f0bfc02a450 comp rx=0 tx=0).stop 2024-09-21T01:46:34.609 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.608+0000 7f0c11f1b640 1 -- 172.21.15.105:0/3976473814 shutdown_connections 2024-09-21T01:46:34.609 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.608+0000 7f0c11f1b640 1 --2- 172.21.15.105:0/3976473814 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0c0c12c610 0x7f0c0c133e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.609 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.608+0000 7f0c11f1b640 1 --2- 172.21.15.105:0/3976473814 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0c0c0733e0 0x7f0c0c0737f0 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.609 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.608+0000 7f0c11f1b640 1 -- 172.21.15.105:0/3976473814 >> 172.21.15.105:0/3976473814 conn(0x7f0c0c09b7c0 msgr2=0x7f0c0c09bbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:34.609 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.608+0000 7f0c11f1b640 1 -- 172.21.15.105:0/3976473814 shutdown_connections 2024-09-21T01:46:34.609 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.608+0000 7f0c11f1b640 1 -- 172.21.15.105:0/3976473814 wait complete. 2024-09-21T01:46:34.610 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.609+0000 7f0c11f1b640 1 Processor -- start 2024-09-21T01:46:34.610 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.609+0000 7f0c11f1b640 1 -- start start 2024-09-21T01:46:34.612 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.610+0000 7f0c11f1b640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c0c0733e0 0x7f0c0c0970a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:34.612 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.610+0000 7f0c11f1b640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0c0c12baf0 0x7f0c0c0975e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:34.612 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.610+0000 7f0c11f1b640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0c0c12c610 0x7f0c0c097b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:34.612 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.610+0000 7f0c11f1b640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f0c0c1343b0 con 0x7f0c0c0733e0 2024-09-21T01:46:34.612 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.610+0000 7f0c11f1b640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f0c0c06fc90 con 0x7f0c0c12baf0 2024-09-21T01:46:34.612 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.610+0000 7f0c11f1b640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f0c0c134530 con 0x7f0c0c12c610 2024-09-21T01:46:34.612 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.610+0000 7f0c0b7fe640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c0c0733e0 0x7f0c0c0970a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:34.612 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.610+0000 7f0c0affd640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0c0c12baf0 0x7f0c0c0975e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:34.612 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.610+0000 7f0c0b7fe640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c0c0733e0 0x7f0c0c0970a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:52232/0 (socket says 172.21.15.105:52232) 2024-09-21T01:46:34.612 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.610+0000 7f0c0bfff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0c0c12c610 0x7f0c0c097b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:34.612 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.610+0000 7f0c0b7fe640 1 -- 172.21.15.105:0/1823445800 learned_addr learned my addr 172.21.15.105:0/1823445800 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:34.612 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.610+0000 7f0c0affd640 1 --2- 172.21.15.105:0/1823445800 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0c0c12baf0 0x7f0c0c0975e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:50764/0 (socket says 172.21.15.105:50764) 2024-09-21T01:46:34.613 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.610+0000 7f0c0b7fe640 1 -- 172.21.15.105:0/1823445800 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0c0c12c610 msgr2=0x7f0c0c097b20 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.613 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.610+0000 7f0c0b7fe640 1 --2- 172.21.15.105:0/1823445800 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0c0c12c610 0x7f0c0c097b20 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.613 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.610+0000 7f0c0b7fe640 1 -- 172.21.15.105:0/1823445800 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0c0c12baf0 msgr2=0x7f0c0c0975e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.613 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.610+0000 7f0c0b7fe640 1 --2- 172.21.15.105:0/1823445800 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0c0c12baf0 0x7f0c0c0975e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.613 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.610+0000 7f0c0b7fe640 1 -- 172.21.15.105:0/1823445800 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f0c0c128820 con 0x7f0c0c0733e0 2024-09-21T01:46:34.613 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.610+0000 7f0c0affd640 1 --2- 172.21.15.105:0/1823445800 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0c0c12baf0 0x7f0c0c0975e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:34.613 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.611+0000 7f0c0b7fe640 1 --2- 172.21.15.105:0/1823445800 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c0c0733e0 0x7f0c0c0970a0 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f0bfc009980 tx=0x7f0bfc033c20 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:34.613 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.611+0000 7f0bf3fff640 1 -- 172.21.15.105:0/1823445800 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0bfc005c90 con 0x7f0c0c0733e0 2024-09-21T01:46:34.613 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.611+0000 7f0bf3fff640 1 -- 172.21.15.105:0/1823445800 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f0bfc031e50 con 0x7f0c0c0733e0 2024-09-21T01:46:34.613 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.611+0000 7f0c11f1b640 1 -- 172.21.15.105:0/1823445800 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f0c0c127f80 con 0x7f0c0c0733e0 2024-09-21T01:46:34.613 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.611+0000 7f0bf3fff640 1 -- 172.21.15.105:0/1823445800 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0bfc005260 con 0x7f0c0c0733e0 2024-09-21T01:46:34.613 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.611+0000 7f0c11f1b640 1 -- 172.21.15.105:0/1823445800 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f0c0c1366e0 con 0x7f0c0c0733e0 2024-09-21T01:46:34.614 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.612+0000 7f0c11f1b640 1 -- 172.21.15.105:0/1823445800 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0c0c137820 con 0x7f0c0c0733e0 2024-09-21T01:46:34.614 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.612+0000 7f0bf3fff640 1 -- 172.21.15.105:0/1823445800 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f0bfc005400 con 0x7f0c0c0733e0 2024-09-21T01:46:34.614 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.612+0000 7f0bf3fff640 1 -- 172.21.15.105:0/1823445800 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f0bfc089f00 con 0x7f0c0c0733e0 2024-09-21T01:46:34.616 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.616+0000 7f0bf3fff640 1 -- 172.21.15.105:0/1823445800 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f0bfc03f030 con 0x7f0c0c0733e0 2024-09-21T01:46:34.763 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.763+0000 7f0c11f1b640 1 -- 172.21.15.105:0/1823445800 --> v1:172.21.15.105:6816/2838005468 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f0c0c137f20 con 0x7f0bf80514b0 2024-09-21T01:46:34.764 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.764+0000 7f0bf3fff640 1 -- 172.21.15.105:0/1823445800 <== mgr.14125 v1:172.21.15.105:6816/2838005468 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32025 (unknown 2965378022 0 3847663262) 0x7f0c0c137f20 con 0x7f0bf80514b0 2024-09-21T01:46:34.765 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:34.765 INFO:teuthology.orchestra.run.smithi105.stderr:dumped all 2024-09-21T01:46:34.767 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.766+0000 7f0c11f1b640 1 -- 172.21.15.105:0/1823445800 >> v1:172.21.15.105:6816/2838005468 conn(0x7f0bf80514b0 legacy=0x7f0bf80539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.767 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.766+0000 7f0c11f1b640 1 -- 172.21.15.105:0/1823445800 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c0c0733e0 msgr2=0x7f0c0c0970a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.767 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.766+0000 7f0c11f1b640 1 --2- 172.21.15.105:0/1823445800 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c0c0733e0 0x7f0c0c0970a0 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f0bfc009980 tx=0x7f0bfc033c20 comp rx=0 tx=0).stop 2024-09-21T01:46:34.767 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.766+0000 7f0c11f1b640 1 -- 172.21.15.105:0/1823445800 shutdown_connections 2024-09-21T01:46:34.767 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.766+0000 7f0c11f1b640 1 --2- 172.21.15.105:0/1823445800 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0c0c12c610 0x7f0c0c097b20 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.767 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.766+0000 7f0c11f1b640 1 --2- 172.21.15.105:0/1823445800 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0c0c12baf0 0x7f0c0c0975e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.767 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.766+0000 7f0c11f1b640 1 --2- 172.21.15.105:0/1823445800 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0c0c0733e0 0x7f0c0c0970a0 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.767 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.766+0000 7f0c11f1b640 1 -- 172.21.15.105:0/1823445800 >> 172.21.15.105:0/1823445800 conn(0x7f0c0c09b7c0 msgr2=0x7f0c0c098ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:34.767 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.766+0000 7f0c11f1b640 1 -- 172.21.15.105:0/1823445800 shutdown_connections 2024-09-21T01:46:34.767 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.766+0000 7f0c11f1b640 1 -- 172.21.15.105:0/1823445800 wait complete. 2024-09-21T01:46:34.778 INFO:teuthology.orchestra.run.smithi105.stdout:{"pg_ready":true,"pg_map":{"version":26,"stamp":"2024-09-21T01:46:33.501143+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112348,"kb_used_data":2256,"kb_used_omap":17,"kb_used_meta":109998,"kb_avail":377375012,"statfs":{"total":386547056640,"available":386432012288,"internally_reserved":0,"allocated":2310144,"data_stored":1569618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17745,"internal_metadata":112638639},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.856498"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":49,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064136+0000","last_change":"2024-09-21T01:46:28.651690+0000","last_active":"2024-09-21T01:46:32.064136+0000","last_peered":"2024-09-21T01:46:32.064136+0000","last_clean":"2024-09-21T01:46:32.064136+0000","last_became_active":"2024-09-21T01:46:28.651383+0000","last_became_peered":"2024-09-21T01:46:28.651383+0000","last_unstale":"2024-09-21T01:46:32.064136+0000","last_undegraded":"2024-09-21T01:46:32.064136+0000","last_fullsized":"2024-09-21T01:46:32.064136+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T01:49:28.532164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040868700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":39,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657657+0000","last_change":"2024-09-21T01:46:26.623500+0000","last_active":"2024-09-21T01:46:31.657657+0000","last_peered":"2024-09-21T01:46:31.657657+0000","last_clean":"2024-09-21T01:46:31.657657+0000","last_became_active":"2024-09-21T01:46:26.623142+0000","last_became_peered":"2024-09-21T01:46:26.623142+0000","last_unstale":"2024-09-21T01:46:31.657657+0000","last_undegraded":"2024-09-21T01:46:31.657657+0000","last_fullsized":"2024-09-21T01:46:31.657657+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:05:55.215381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055542700000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":49,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064256+0000","last_change":"2024-09-21T01:46:28.649216+0000","last_active":"2024-09-21T01:46:32.064256+0000","last_peered":"2024-09-21T01:46:32.064256+0000","last_clean":"2024-09-21T01:46:32.064256+0000","last_became_active":"2024-09-21T01:46:28.648886+0000","last_became_peered":"2024-09-21T01:46:28.648886+0000","last_unstale":"2024-09-21T01:46:32.064256+0000","last_undegraded":"2024-09-21T01:46:32.064256+0000","last_fullsized":"2024-09-21T01:46:32.064256+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:09:06.681843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00119484,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":39,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657839+0000","last_change":"2024-09-21T01:46:26.624399+0000","last_active":"2024-09-21T01:46:31.657839+0000","last_peered":"2024-09-21T01:46:31.657839+0000","last_clean":"2024-09-21T01:46:31.657839+0000","last_became_active":"2024-09-21T01:46:26.624073+0000","last_became_peered":"2024-09-21T01:46:26.624073+0000","last_unstale":"2024-09-21T01:46:31.657839+0000","last_undegraded":"2024-09-21T01:46:31.657839+0000","last_fullsized":"2024-09-21T01:46:31.657839+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T08:18:28.384798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063215099999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":46,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064137+0000","last_change":"2024-09-21T01:46:28.654323+0000","last_active":"2024-09-21T01:46:32.064137+0000","last_peered":"2024-09-21T01:46:32.064137+0000","last_clean":"2024-09-21T01:46:32.064137+0000","last_became_active":"2024-09-21T01:46:28.653984+0000","last_became_peered":"2024-09-21T01:46:28.653984+0000","last_unstale":"2024-09-21T01:46:32.064137+0000","last_undegraded":"2024-09-21T01:46:32.064137+0000","last_fullsized":"2024-09-21T01:46:32.064137+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:41:46.114559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010245149999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":36,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657473+0000","last_change":"2024-09-21T01:46:28.656954+0000","last_active":"2024-09-21T01:46:31.657473+0000","last_peered":"2024-09-21T01:46:31.657473+0000","last_clean":"2024-09-21T01:46:31.657473+0000","last_became_active":"2024-09-21T01:46:28.656588+0000","last_became_peered":"2024-09-21T01:46:28.656588+0000","last_unstale":"2024-09-21T01:46:31.657473+0000","last_undegraded":"2024-09-21T01:46:31.657473+0000","last_fullsized":"2024-09-21T01:46:31.657473+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T02:10:00.211563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080666500000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":44,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064104+0000","last_change":"2024-09-21T01:46:28.657038+0000","last_active":"2024-09-21T01:46:32.064104+0000","last_peered":"2024-09-21T01:46:32.064104+0000","last_clean":"2024-09-21T01:46:32.064104+0000","last_became_active":"2024-09-21T01:46:28.656701+0000","last_became_peered":"2024-09-21T01:46:28.656701+0000","last_unstale":"2024-09-21T01:46:32.064104+0000","last_undegraded":"2024-09-21T01:46:32.064104+0000","last_fullsized":"2024-09-21T01:46:32.064104+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T05:11:34.297379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058759600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":42,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064133+0000","last_change":"2024-09-21T01:46:28.657189+0000","last_active":"2024-09-21T01:46:32.064133+0000","last_peered":"2024-09-21T01:46:32.064133+0000","last_clean":"2024-09-21T01:46:32.064133+0000","last_became_active":"2024-09-21T01:46:28.656728+0000","last_became_peered":"2024-09-21T01:46:28.656728+0000","last_unstale":"2024-09-21T01:46:32.064133+0000","last_undegraded":"2024-09-21T01:46:32.064133+0000","last_fullsized":"2024-09-21T01:46:32.064133+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T08:49:15.772867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036256899999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"20'104","reported_seq":189,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657393+0000","last_change":"2024-09-21T01:46:26.626044+0000","last_active":"2024-09-21T01:46:31.657393+0000","last_peered":"2024-09-21T01:46:31.657393+0000","last_clean":"2024-09-21T01:46:31.657393+0000","last_became_active":"2024-09-21T01:46:26.625434+0000","last_became_peered":"2024-09-21T01:46:26.625434+0000","last_unstale":"2024-09-21T01:46:31.657393+0000","last_undegraded":"2024-09-21T01:46:31.657393+0000","last_fullsized":"2024-09-21T01:46:31.657393+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:50.489745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:50.489745+0000","last_clean_scrub_stamp":"2024-09-21T01:44:50.489745+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T08:39:55.782994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":29,"seq":124554051587,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27816,"kb_used_data":276,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344024,"statfs":{"total":96636764160,"available":96608280576,"internally_reserved":0,"allocated":282624,"data_stored":97230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4435,"internal_metadata":28176045},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":29,"seq":124554051587,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27812,"kb_used_data":272,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344028,"statfs":{"total":96636764160,"available":96608284672,"internally_reserved":0,"allocated":278528,"data_stored":97211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28176043},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":27,"seq":115964116995,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28328,"kb_used_data":856,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343512,"statfs":{"total":96636764160,"available":96607756288,"internally_reserved":0,"allocated":876544,"data_stored":687598,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4436,"internal_metadata":28110508},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":26,"seq":111669149699,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28392,"kb_used_data":852,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94343448,"statfs":{"total":96636764160,"available":96607690752,"internally_reserved":0,"allocated":872448,"data_stored":687579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28176043},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T01:46:34.779 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T01:46:34.888 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.888+0000 7f8dd6158640 1 -- 172.21.15.105:0/3648581313 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8dd009bce0 msgr2=0x7f8dd012c8a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.889 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.888+0000 7f8dd6158640 1 --2- 172.21.15.105:0/3648581313 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8dd009bce0 0x7f8dd012c8a0 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f8dc0009860 tx=0x7f8dc002a3d0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.889 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.888+0000 7f8dd6158640 1 -- 172.21.15.105:0/3648581313 shutdown_connections 2024-09-21T01:46:34.889 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.888+0000 7f8dd6158640 1 --2- 172.21.15.105:0/3648581313 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8dd0071f70 0x7f8dd0134830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.889 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.888+0000 7f8dd6158640 1 --2- 172.21.15.105:0/3648581313 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8dd009bce0 0x7f8dd012c8a0 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.889 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.888+0000 7f8dd6158640 1 -- 172.21.15.105:0/3648581313 >> 172.21.15.105:0/3648581313 conn(0x7f8dd009f000 msgr2=0x7f8dd009b4d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:34.889 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.888+0000 7f8dd6158640 1 -- 172.21.15.105:0/3648581313 shutdown_connections 2024-09-21T01:46:34.889 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.888+0000 7f8dd6158640 1 -- 172.21.15.105:0/3648581313 wait complete. 2024-09-21T01:46:34.891 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.889+0000 7f8dd6158640 1 Processor -- start 2024-09-21T01:46:34.892 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.889+0000 7f8dd6158640 1 -- start start 2024-09-21T01:46:34.892 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.890+0000 7f8dd6158640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8dd0071f70 0x7f8dd01307c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:34.892 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.890+0000 7f8dd6158640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8dd009bce0 0x7f8dd0130d00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:34.892 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.890+0000 7f8dd6158640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8dd012cde0 0x7f8dd0131240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:34.892 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.890+0000 7f8dd6158640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f8dd0135500 con 0x7f8dd009bce0 2024-09-21T01:46:34.892 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.890+0000 7f8dd6158640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f8dd0135380 con 0x7f8dd0071f70 2024-09-21T01:46:34.892 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.890+0000 7f8dd6158640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f8dd0135680 con 0x7f8dd012cde0 2024-09-21T01:46:34.892 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.890+0000 7f8dcf7fe640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8dd0071f70 0x7f8dd01307c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:34.892 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.890+0000 7f8dceffd640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8dd009bce0 0x7f8dd0130d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:34.892 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.890+0000 7f8dcf7fe640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8dd0071f70 0x7f8dd01307c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:50778/0 (socket says 172.21.15.105:50778) 2024-09-21T01:46:34.892 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.890+0000 7f8dcf7fe640 1 -- 172.21.15.105:0/542204898 learned_addr learned my addr 172.21.15.105:0/542204898 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:34.892 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.890+0000 7f8dcffff640 1 --2- 172.21.15.105:0/542204898 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8dd012cde0 0x7f8dd0131240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:34.892 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.890+0000 7f8dcf7fe640 1 -- 172.21.15.105:0/542204898 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8dd012cde0 msgr2=0x7f8dd0131240 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.892 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.890+0000 7f8dcf7fe640 1 --2- 172.21.15.105:0/542204898 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8dd012cde0 0x7f8dd0131240 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.893 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.890+0000 7f8dcf7fe640 1 -- 172.21.15.105:0/542204898 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8dd009bce0 msgr2=0x7f8dd0130d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:34.893 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.890+0000 7f8dcf7fe640 1 --2- 172.21.15.105:0/542204898 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8dd009bce0 0x7f8dd0130d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:34.893 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.890+0000 7f8dcf7fe640 1 -- 172.21.15.105:0/542204898 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8dd0132860 con 0x7f8dd0071f70 2024-09-21T01:46:34.893 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.890+0000 7f8dcffff640 1 --2- 172.21.15.105:0/542204898 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8dd012cde0 0x7f8dd0131240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:34.893 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.890+0000 7f8dceffd640 1 --2- 172.21.15.105:0/542204898 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8dd009bce0 0x7f8dd0130d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:34.893 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.890+0000 7f8dcf7fe640 1 --2- 172.21.15.105:0/542204898 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8dd0071f70 0x7f8dd01307c0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f8dc0009780 tx=0x7f8dc0032c10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:34.893 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.891+0000 7f8db7fff640 1 -- 172.21.15.105:0/542204898 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8dc0040070 con 0x7f8dd0071f70 2024-09-21T01:46:34.893 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.891+0000 7f8db7fff640 1 -- 172.21.15.105:0/542204898 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f8dc0005310 con 0x7f8dd0071f70 2024-09-21T01:46:34.893 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.891+0000 7f8db7fff640 1 -- 172.21.15.105:0/542204898 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8dc00047a0 con 0x7f8dd0071f70 2024-09-21T01:46:34.893 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.891+0000 7f8dd6158640 1 -- 172.21.15.105:0/542204898 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8dd0133060 con 0x7f8dd0071f70 2024-09-21T01:46:34.893 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.891+0000 7f8dd6158640 1 -- 172.21.15.105:0/542204898 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f8dd0098290 con 0x7f8dd0071f70 2024-09-21T01:46:34.894 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.892+0000 7f8dd6158640 1 -- 172.21.15.105:0/542204898 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8dd012c8a0 con 0x7f8dd0071f70 2024-09-21T01:46:34.894 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.892+0000 7f8db7fff640 1 -- 172.21.15.105:0/542204898 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f8dc0039040 con 0x7f8dd0071f70 2024-09-21T01:46:34.894 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.892+0000 7f8db7fff640 1 -- 172.21.15.105:0/542204898 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f8dc008a250 con 0x7f8dd0071f70 2024-09-21T01:46:34.896 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:34.896+0000 7f8db7fff640 1 -- 172.21.15.105:0/542204898 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f8dc0089300 con 0x7f8dd0071f70 2024-09-21T01:46:35.042 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.042+0000 7f8dd6158640 1 -- 172.21.15.105:0/542204898 --> v1:172.21.15.105:6816/2838005468 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f8dd006a820 con 0x7f8dbc0514b0 2024-09-21T01:46:35.043 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.043+0000 7f8db7fff640 1 -- 172.21.15.105:0/542204898 <== mgr.14125 v1:172.21.15.105:6816/2838005468 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32025 (unknown 2965378022 0 3847663262) 0x7f8dd006a820 con 0x7f8dbc0514b0 2024-09-21T01:46:35.044 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:35.044 INFO:teuthology.orchestra.run.smithi105.stderr:dumped all 2024-09-21T01:46:35.045 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.045+0000 7f8dd6158640 1 -- 172.21.15.105:0/542204898 >> v1:172.21.15.105:6816/2838005468 conn(0x7f8dbc0514b0 legacy=0x7f8dbc0539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:35.045 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.045+0000 7f8dd6158640 1 -- 172.21.15.105:0/542204898 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8dd0071f70 msgr2=0x7f8dd01307c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:35.045 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.045+0000 7f8dd6158640 1 --2- 172.21.15.105:0/542204898 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8dd0071f70 0x7f8dd01307c0 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f8dc0009780 tx=0x7f8dc0032c10 comp rx=0 tx=0).stop 2024-09-21T01:46:35.045 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.045+0000 7f8dd6158640 1 -- 172.21.15.105:0/542204898 shutdown_connections 2024-09-21T01:46:35.046 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.045+0000 7f8dd6158640 1 --2- 172.21.15.105:0/542204898 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8dd012cde0 0x7f8dd0131240 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:35.046 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.045+0000 7f8dd6158640 1 --2- 172.21.15.105:0/542204898 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8dd009bce0 0x7f8dd0130d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:35.046 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.045+0000 7f8dd6158640 1 --2- 172.21.15.105:0/542204898 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8dd0071f70 0x7f8dd01307c0 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:35.046 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.045+0000 7f8dd6158640 1 -- 172.21.15.105:0/542204898 >> 172.21.15.105:0/542204898 conn(0x7f8dd009f000 msgr2=0x7f8dd009ac20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:35.046 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.045+0000 7f8dd6158640 1 -- 172.21.15.105:0/542204898 shutdown_connections 2024-09-21T01:46:35.046 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.045+0000 7f8dd6158640 1 -- 172.21.15.105:0/542204898 wait complete. 2024-09-21T01:46:35.056 INFO:teuthology.orchestra.run.smithi105.stdout:{"pg_ready":true,"pg_map":{"version":26,"stamp":"2024-09-21T01:46:33.501143+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112348,"kb_used_data":2256,"kb_used_omap":17,"kb_used_meta":109998,"kb_avail":377375012,"statfs":{"total":386547056640,"available":386432012288,"internally_reserved":0,"allocated":2310144,"data_stored":1569618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17745,"internal_metadata":112638639},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.856498"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":49,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064136+0000","last_change":"2024-09-21T01:46:28.651690+0000","last_active":"2024-09-21T01:46:32.064136+0000","last_peered":"2024-09-21T01:46:32.064136+0000","last_clean":"2024-09-21T01:46:32.064136+0000","last_became_active":"2024-09-21T01:46:28.651383+0000","last_became_peered":"2024-09-21T01:46:28.651383+0000","last_unstale":"2024-09-21T01:46:32.064136+0000","last_undegraded":"2024-09-21T01:46:32.064136+0000","last_fullsized":"2024-09-21T01:46:32.064136+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T01:49:28.532164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040868700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":39,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657657+0000","last_change":"2024-09-21T01:46:26.623500+0000","last_active":"2024-09-21T01:46:31.657657+0000","last_peered":"2024-09-21T01:46:31.657657+0000","last_clean":"2024-09-21T01:46:31.657657+0000","last_became_active":"2024-09-21T01:46:26.623142+0000","last_became_peered":"2024-09-21T01:46:26.623142+0000","last_unstale":"2024-09-21T01:46:31.657657+0000","last_undegraded":"2024-09-21T01:46:31.657657+0000","last_fullsized":"2024-09-21T01:46:31.657657+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:05:55.215381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055542700000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":49,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064256+0000","last_change":"2024-09-21T01:46:28.649216+0000","last_active":"2024-09-21T01:46:32.064256+0000","last_peered":"2024-09-21T01:46:32.064256+0000","last_clean":"2024-09-21T01:46:32.064256+0000","last_became_active":"2024-09-21T01:46:28.648886+0000","last_became_peered":"2024-09-21T01:46:28.648886+0000","last_unstale":"2024-09-21T01:46:32.064256+0000","last_undegraded":"2024-09-21T01:46:32.064256+0000","last_fullsized":"2024-09-21T01:46:32.064256+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:09:06.681843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00119484,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":39,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657839+0000","last_change":"2024-09-21T01:46:26.624399+0000","last_active":"2024-09-21T01:46:31.657839+0000","last_peered":"2024-09-21T01:46:31.657839+0000","last_clean":"2024-09-21T01:46:31.657839+0000","last_became_active":"2024-09-21T01:46:26.624073+0000","last_became_peered":"2024-09-21T01:46:26.624073+0000","last_unstale":"2024-09-21T01:46:31.657839+0000","last_undegraded":"2024-09-21T01:46:31.657839+0000","last_fullsized":"2024-09-21T01:46:31.657839+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T08:18:28.384798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063215099999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":46,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064137+0000","last_change":"2024-09-21T01:46:28.654323+0000","last_active":"2024-09-21T01:46:32.064137+0000","last_peered":"2024-09-21T01:46:32.064137+0000","last_clean":"2024-09-21T01:46:32.064137+0000","last_became_active":"2024-09-21T01:46:28.653984+0000","last_became_peered":"2024-09-21T01:46:28.653984+0000","last_unstale":"2024-09-21T01:46:32.064137+0000","last_undegraded":"2024-09-21T01:46:32.064137+0000","last_fullsized":"2024-09-21T01:46:32.064137+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:41:46.114559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010245149999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":36,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657473+0000","last_change":"2024-09-21T01:46:28.656954+0000","last_active":"2024-09-21T01:46:31.657473+0000","last_peered":"2024-09-21T01:46:31.657473+0000","last_clean":"2024-09-21T01:46:31.657473+0000","last_became_active":"2024-09-21T01:46:28.656588+0000","last_became_peered":"2024-09-21T01:46:28.656588+0000","last_unstale":"2024-09-21T01:46:31.657473+0000","last_undegraded":"2024-09-21T01:46:31.657473+0000","last_fullsized":"2024-09-21T01:46:31.657473+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T02:10:00.211563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080666500000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":44,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064104+0000","last_change":"2024-09-21T01:46:28.657038+0000","last_active":"2024-09-21T01:46:32.064104+0000","last_peered":"2024-09-21T01:46:32.064104+0000","last_clean":"2024-09-21T01:46:32.064104+0000","last_became_active":"2024-09-21T01:46:28.656701+0000","last_became_peered":"2024-09-21T01:46:28.656701+0000","last_unstale":"2024-09-21T01:46:32.064104+0000","last_undegraded":"2024-09-21T01:46:32.064104+0000","last_fullsized":"2024-09-21T01:46:32.064104+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T05:11:34.297379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058759600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":42,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064133+0000","last_change":"2024-09-21T01:46:28.657189+0000","last_active":"2024-09-21T01:46:32.064133+0000","last_peered":"2024-09-21T01:46:32.064133+0000","last_clean":"2024-09-21T01:46:32.064133+0000","last_became_active":"2024-09-21T01:46:28.656728+0000","last_became_peered":"2024-09-21T01:46:28.656728+0000","last_unstale":"2024-09-21T01:46:32.064133+0000","last_undegraded":"2024-09-21T01:46:32.064133+0000","last_fullsized":"2024-09-21T01:46:32.064133+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T08:49:15.772867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036256899999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"20'104","reported_seq":189,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657393+0000","last_change":"2024-09-21T01:46:26.626044+0000","last_active":"2024-09-21T01:46:31.657393+0000","last_peered":"2024-09-21T01:46:31.657393+0000","last_clean":"2024-09-21T01:46:31.657393+0000","last_became_active":"2024-09-21T01:46:26.625434+0000","last_became_peered":"2024-09-21T01:46:26.625434+0000","last_unstale":"2024-09-21T01:46:31.657393+0000","last_undegraded":"2024-09-21T01:46:31.657393+0000","last_fullsized":"2024-09-21T01:46:31.657393+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:50.489745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:50.489745+0000","last_clean_scrub_stamp":"2024-09-21T01:44:50.489745+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T08:39:55.782994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":29,"seq":124554051587,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27816,"kb_used_data":276,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344024,"statfs":{"total":96636764160,"available":96608280576,"internally_reserved":0,"allocated":282624,"data_stored":97230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4435,"internal_metadata":28176045},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":29,"seq":124554051587,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27812,"kb_used_data":272,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344028,"statfs":{"total":96636764160,"available":96608284672,"internally_reserved":0,"allocated":278528,"data_stored":97211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28176043},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":27,"seq":115964116995,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28328,"kb_used_data":856,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343512,"statfs":{"total":96636764160,"available":96607756288,"internally_reserved":0,"allocated":876544,"data_stored":687598,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4436,"internal_metadata":28110508},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":26,"seq":111669149699,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28392,"kb_used_data":852,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94343448,"statfs":{"total":96636764160,"available":96607690752,"internally_reserved":0,"allocated":872448,"data_stored":687579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28176043},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T01:46:35.057 INFO:tasks.ceph.ceph_manager.ceph:clean! 2024-09-21T01:46:35.057 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T01:46:35.057 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy 2024-09-21T01:46:35.057 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph health --format=json 2024-09-21T01:46:35.166 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.165+0000 7f44f8165640 1 -- 172.21.15.105:0/4112445958 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f44f012c8f0 msgr2=0x7f44f012cd00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:35.166 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.165+0000 7f44f8165640 1 --2- 172.21.15.105:0/4112445958 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f44f012c8f0 0x7f44f012cd00 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7f44e0009830 tx=0x7f44e002a3f0 comp rx=0 tx=0).stop 2024-09-21T01:46:35.166 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.165+0000 7f44f8165640 1 -- 172.21.15.105:0/4112445958 shutdown_connections 2024-09-21T01:46:35.166 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.165+0000 7f44f8165640 1 --2- 172.21.15.105:0/4112445958 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f44f0128d40 0x7f44f0129150 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:35.166 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.165+0000 7f44f8165640 1 --2- 172.21.15.105:0/4112445958 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f44f012c8f0 0x7f44f012cd00 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:35.166 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.165+0000 7f44f8165640 1 --2- 172.21.15.105:0/4112445958 >> v2:172.21.15.105:3300/0 conn(0x7f44f00729c0 0x7f44f012c3b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:35.166 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.165+0000 7f44f8165640 1 -- 172.21.15.105:0/4112445958 >> 172.21.15.105:0/4112445958 conn(0x7f44f009b7c0 msgr2=0x7f44f009bbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:35.166 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.165+0000 7f44f8165640 1 -- 172.21.15.105:0/4112445958 shutdown_connections 2024-09-21T01:46:35.166 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.165+0000 7f44f8165640 1 -- 172.21.15.105:0/4112445958 wait complete. 2024-09-21T01:46:35.168 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.166+0000 7f44f8165640 1 Processor -- start 2024-09-21T01:46:35.168 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.166+0000 7f44f8165640 1 -- start start 2024-09-21T01:46:35.168 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.167+0000 7f44f8165640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f44f0128d40 0x7f44f01c3700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:35.169 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.167+0000 7f44f8165640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f44f012c8f0 0x7f44f01c3c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:35.169 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.167+0000 7f44f8165640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f44f01c4180 0x7f44f012d300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:35.169 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.167+0000 7f44f8165640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f44f00709a0 con 0x7f44f01c4180 2024-09-21T01:46:35.169 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.167+0000 7f44f8165640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f44f0070820 con 0x7f44f012c8f0 2024-09-21T01:46:35.169 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.167+0000 7f44f8165640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f44f0070b20 con 0x7f44f0128d40 2024-09-21T01:46:35.169 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.167+0000 7f44f5f01640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f44f0128d40 0x7f44f01c3700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:35.169 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.167+0000 7f44f5f01640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f44f0128d40 0x7f44f01c3700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:48528/0 (socket says 172.21.15.105:48528) 2024-09-21T01:46:35.169 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.167+0000 7f44f5700640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f44f012c8f0 0x7f44f01c3c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:35.169 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.167+0000 7f44f5f01640 1 -- 172.21.15.105:0/1702081678 learned_addr learned my addr 172.21.15.105:0/1702081678 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:35.169 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.167+0000 7f44f6702640 1 --2- 172.21.15.105:0/1702081678 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f44f01c4180 0x7f44f012d300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:35.169 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.167+0000 7f44f5f01640 1 -- 172.21.15.105:0/1702081678 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f44f012c8f0 msgr2=0x7f44f01c3c40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:35.169 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.167+0000 7f44f5f01640 1 --2- 172.21.15.105:0/1702081678 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f44f012c8f0 0x7f44f01c3c40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:35.169 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.167+0000 7f44f5f01640 1 -- 172.21.15.105:0/1702081678 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f44f01c4180 msgr2=0x7f44f012d300 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:35.169 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.167+0000 7f44f5f01640 1 --2- 172.21.15.105:0/1702081678 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f44f01c4180 0x7f44f012d300 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:35.170 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.167+0000 7f44f5f01640 1 -- 172.21.15.105:0/1702081678 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f44f0130320 con 0x7f44f0128d40 2024-09-21T01:46:35.170 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.167+0000 7f44f5f01640 1 --2- 172.21.15.105:0/1702081678 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f44f0128d40 0x7f44f01c3700 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f44ec00e9c0 tx=0x7f44ec00edc0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:35.170 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.168+0000 7f44e67fc640 1 -- 172.21.15.105:0/1702081678 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f44ec00ccc0 con 0x7f44f0128d40 2024-09-21T01:46:35.170 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.168+0000 7f44e67fc640 1 -- 172.21.15.105:0/1702081678 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f44ec004020 con 0x7f44f0128d40 2024-09-21T01:46:35.170 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.168+0000 7f44f8165640 1 -- 172.21.15.105:0/1702081678 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f44f012fcd0 con 0x7f44f0128d40 2024-09-21T01:46:35.170 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.168+0000 7f44e67fc640 1 -- 172.21.15.105:0/1702081678 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f44ec00ce60 con 0x7f44f0128d40 2024-09-21T01:46:35.170 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.168+0000 7f44f8165640 1 -- 172.21.15.105:0/1702081678 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f44f012da20 con 0x7f44f0128d40 2024-09-21T01:46:35.170 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.169+0000 7f44f8165640 1 -- 172.21.15.105:0/1702081678 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f44f012e680 con 0x7f44f0128d40 2024-09-21T01:46:35.171 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.169+0000 7f44e67fc640 1 -- 172.21.15.105:0/1702081678 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f44ec0035f0 con 0x7f44f0128d40 2024-09-21T01:46:35.171 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.169+0000 7f44e67fc640 1 -- 172.21.15.105:0/1702081678 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f44ec069850 con 0x7f44f0128d40 2024-09-21T01:46:35.173 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.173+0000 7f44e67fc640 1 -- 172.21.15.105:0/1702081678 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f44ec004450 con 0x7f44f0128d40 2024-09-21T01:46:35.348 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.347+0000 7f44f8165640 1 -- 172.21.15.105:0/1702081678 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "health", "format": "json"} v 0) -- 0x7f44f00729c0 con 0x7f44f0128d40 2024-09-21T01:46:35.348 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.348+0000 7f44e67fc640 1 -- 172.21.15.105:0/1702081678 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "health", "format": "json"}]=0 v0) ==== 72+0+46 (secure 0 0 0) 0x7f44ec068900 con 0x7f44f0128d40 2024-09-21T01:46:35.348 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:35.348 INFO:teuthology.orchestra.run.smithi105.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T01:46:35.350 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.350+0000 7f44f8165640 1 -- 172.21.15.105:0/1702081678 >> v1:172.21.15.105:6816/2838005468 conn(0x7f44d40514b0 legacy=0x7f44d40539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:35.350 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.350+0000 7f44f8165640 1 -- 172.21.15.105:0/1702081678 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f44f0128d40 msgr2=0x7f44f01c3700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:35.350 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.350+0000 7f44f8165640 1 --2- 172.21.15.105:0/1702081678 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f44f0128d40 0x7f44f01c3700 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f44ec00e9c0 tx=0x7f44ec00edc0 comp rx=0 tx=0).stop 2024-09-21T01:46:35.351 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.350+0000 7f44f8165640 1 -- 172.21.15.105:0/1702081678 shutdown_connections 2024-09-21T01:46:35.351 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.350+0000 7f44f8165640 1 --2- 172.21.15.105:0/1702081678 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f44f01c4180 0x7f44f012d300 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:35.351 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.350+0000 7f44f8165640 1 --2- 172.21.15.105:0/1702081678 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f44f012c8f0 0x7f44f01c3c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:35.351 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.350+0000 7f44f8165640 1 --2- 172.21.15.105:0/1702081678 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f44f0128d40 0x7f44f01c3700 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:35.351 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.350+0000 7f44f8165640 1 -- 172.21.15.105:0/1702081678 >> 172.21.15.105:0/1702081678 conn(0x7f44f009b7c0 msgr2=0x7f44f0097550 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:35.351 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.350+0000 7f44f8165640 1 -- 172.21.15.105:0/1702081678 shutdown_connections 2024-09-21T01:46:35.351 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.350+0000 7f44f8165640 1 -- 172.21.15.105:0/1702081678 wait complete. 2024-09-21T01:46:35.361 INFO:tasks.ceph.ceph_manager.ceph:wait_until_healthy done 2024-09-21T01:46:35.362 INFO:teuthology.run_tasks:Running task print... 2024-09-21T01:46:35.372 INFO:teuthology.task.print:**** done ceph.restart 2024-09-21T01:46:35.372 DEBUG:teuthology.run_tasks:Unwinding manager ceph.restart 2024-09-21T01:46:35.445 DEBUG:teuthology.run_tasks:Unwinding manager ceph.restart 2024-09-21T01:46:35.531 DEBUG:teuthology.run_tasks:Unwinding manager install.upgrade 2024-09-21T01:46:35.615 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2024-09-21T01:46:35.689 INFO:tasks.ceph.ceph_manager.ceph:waiting for clean 2024-09-21T01:46:35.689 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T01:46:35.800 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.799+0000 7f4011c34640 1 -- 172.21.15.105:0/1369967747 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f400c12e020 msgr2=0x7f400c1327a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:35.800 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.799+0000 7f4011c34640 1 --2- 172.21.15.105:0/1369967747 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f400c12e020 0x7f400c1327a0 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f4000005fd0 tx=0x7f400002a540 comp rx=0 tx=0).stop 2024-09-21T01:46:35.800 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.799+0000 7f4011c34640 1 -- 172.21.15.105:0/1369967747 shutdown_connections 2024-09-21T01:46:35.800 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.799+0000 7f4011c34640 1 --2- 172.21.15.105:0/1369967747 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f400c132ce0 0x7f400c135100 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:35.800 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.799+0000 7f4011c34640 1 --2- 172.21.15.105:0/1369967747 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f400c12e020 0x7f400c1327a0 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:35.801 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.799+0000 7f4011c34640 1 -- 172.21.15.105:0/1369967747 >> 172.21.15.105:0/1369967747 conn(0x7f400c09fe70 msgr2=0x7f400c0a0280 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:35.801 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.799+0000 7f4011c34640 1 -- 172.21.15.105:0/1369967747 shutdown_connections 2024-09-21T01:46:35.801 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.799+0000 7f4011c34640 1 -- 172.21.15.105:0/1369967747 wait complete. 2024-09-21T01:46:35.803 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.800+0000 7f4011c34640 1 Processor -- start 2024-09-21T01:46:35.803 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.801+0000 7f4011c34640 1 -- start start 2024-09-21T01:46:35.803 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.801+0000 7f4011c34640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f400c073b30 0x7f400c12e4c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:35.803 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.801+0000 7f4011c34640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f400c12e020 0x7f400c12ea00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:35.803 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.801+0000 7f4011c34640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f400c132ce0 0x7f400c12ef40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:35.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.801+0000 7f4011c34640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f400c1364b0 con 0x7f400c132ce0 2024-09-21T01:46:35.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.801+0000 7f4011c34640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f400c136330 con 0x7f400c12e020 2024-09-21T01:46:35.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.801+0000 7f4011c34640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f400c136630 con 0x7f400c073b30 2024-09-21T01:46:35.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.801+0000 7f400b7fe640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f400c073b30 0x7f400c12e4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:35.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.801+0000 7f400bfff640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f400c132ce0 0x7f400c12ef40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:35.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.801+0000 7f400affd640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f400c12e020 0x7f400c12ea00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:35.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.801+0000 7f400bfff640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f400c132ce0 0x7f400c12ef40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:52272/0 (socket says 172.21.15.105:52272) 2024-09-21T01:46:35.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.801+0000 7f400bfff640 1 -- 172.21.15.105:0/2915269441 learned_addr learned my addr 172.21.15.105:0/2915269441 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:35.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.801+0000 7f400b7fe640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f400c073b30 0x7f400c12e4c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:48556/0 (socket says 172.21.15.105:48556) 2024-09-21T01:46:35.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.802+0000 7f400affd640 1 -- 172.21.15.105:0/2915269441 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f400c073b30 msgr2=0x7f400c12e4c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:35.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.802+0000 7f400affd640 1 --2- 172.21.15.105:0/2915269441 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f400c073b30 0x7f400c12e4c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:35.805 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.802+0000 7f400affd640 1 -- 172.21.15.105:0/2915269441 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f400c132ce0 msgr2=0x7f400c12ef40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:35.805 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.802+0000 7f400affd640 1 --2- 172.21.15.105:0/2915269441 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f400c132ce0 0x7f400c12ef40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:35.805 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.802+0000 7f400affd640 1 -- 172.21.15.105:0/2915269441 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f400c130610 con 0x7f400c12e020 2024-09-21T01:46:35.805 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.802+0000 7f400b7fe640 1 --2- 172.21.15.105:0/2915269441 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f400c073b30 0x7f400c12e4c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:35.805 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.802+0000 7f400bfff640 1 --2- 172.21.15.105:0/2915269441 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f400c132ce0 0x7f400c12ef40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:35.805 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.802+0000 7f400affd640 1 --2- 172.21.15.105:0/2915269441 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f400c12e020 0x7f400c12ea00 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f4000031000 tx=0x7f40000038a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:35.805 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.802+0000 7f3ff3fff640 1 -- 172.21.15.105:0/2915269441 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f400003f070 con 0x7f400c12e020 2024-09-21T01:46:35.805 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.802+0000 7f3ff3fff640 1 -- 172.21.15.105:0/2915269441 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f40000047a0 con 0x7f400c12e020 2024-09-21T01:46:35.805 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.802+0000 7f4011c34640 1 -- 172.21.15.105:0/2915269441 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f400c131410 con 0x7f400c12e020 2024-09-21T01:46:35.805 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.802+0000 7f3ff3fff640 1 -- 172.21.15.105:0/2915269441 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4000035030 con 0x7f400c12e020 2024-09-21T01:46:35.805 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.802+0000 7f4011c34640 1 -- 172.21.15.105:0/2915269441 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f400c1c3900 con 0x7f400c12e020 2024-09-21T01:46:35.806 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.803+0000 7f4011c34640 1 -- 172.21.15.105:0/2915269441 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f400c12d910 con 0x7f400c12e020 2024-09-21T01:46:35.806 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.803+0000 7f3ff3fff640 1 -- 172.21.15.105:0/2915269441 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f4000042070 con 0x7f400c12e020 2024-09-21T01:46:35.806 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.804+0000 7f3ff3fff640 1 -- 172.21.15.105:0/2915269441 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f4000089e40 con 0x7f400c12e020 2024-09-21T01:46:35.808 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.808+0000 7f3ff3fff640 1 -- 172.21.15.105:0/2915269441 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f4000088ef0 con 0x7f400c12e020 2024-09-21T01:46:35.952 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.952+0000 7f4011c34640 1 -- 172.21.15.105:0/2915269441 --> v1:172.21.15.105:6816/2838005468 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f400c06a740 con 0x7f3ff80514b0 2024-09-21T01:46:35.953 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.953+0000 7f3ff3fff640 1 -- 172.21.15.105:0/2915269441 <== mgr.14125 v1:172.21.15.105:6816/2838005468 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32001 (unknown 2965378022 0 4261581420) 0x7f400c06a740 con 0x7f3ff80514b0 2024-09-21T01:46:35.953 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:35.954 INFO:teuthology.orchestra.run.smithi105.stderr:dumped all 2024-09-21T01:46:35.955 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.954+0000 7f4011c34640 1 -- 172.21.15.105:0/2915269441 >> v1:172.21.15.105:6816/2838005468 conn(0x7f3ff80514b0 legacy=0x7f3ff80539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:35.955 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.954+0000 7f4011c34640 1 -- 172.21.15.105:0/2915269441 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f400c12e020 msgr2=0x7f400c12ea00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:35.955 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.954+0000 7f4011c34640 1 --2- 172.21.15.105:0/2915269441 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f400c12e020 0x7f400c12ea00 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f4000031000 tx=0x7f40000038a0 comp rx=0 tx=0).stop 2024-09-21T01:46:35.955 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.955+0000 7f4011c34640 1 -- 172.21.15.105:0/2915269441 shutdown_connections 2024-09-21T01:46:35.955 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.955+0000 7f4011c34640 1 --2- 172.21.15.105:0/2915269441 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f400c132ce0 0x7f400c12ef40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:35.955 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.955+0000 7f4011c34640 1 --2- 172.21.15.105:0/2915269441 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f400c12e020 0x7f400c12ea00 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:35.956 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.955+0000 7f4011c34640 1 --2- 172.21.15.105:0/2915269441 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f400c073b30 0x7f400c12e4c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:35.956 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.955+0000 7f4011c34640 1 -- 172.21.15.105:0/2915269441 >> 172.21.15.105:0/2915269441 conn(0x7f400c09fe70 msgr2=0x7f400c12a2d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:35.956 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.955+0000 7f4011c34640 1 -- 172.21.15.105:0/2915269441 shutdown_connections 2024-09-21T01:46:35.956 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:35.955+0000 7f4011c34640 1 -- 172.21.15.105:0/2915269441 wait complete. 2024-09-21T01:46:35.966 INFO:teuthology.orchestra.run.smithi105.stdout:{"pg_ready":true,"pg_map":{"version":27,"stamp":"2024-09-21T01:46:35.501509+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112348,"kb_used_data":2256,"kb_used_omap":17,"kb_used_meta":109998,"kb_avail":377375012,"statfs":{"total":386547056640,"available":386432012288,"internally_reserved":0,"allocated":2310144,"data_stored":1569618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17745,"internal_metadata":112638639},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001190"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":49,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064136+0000","last_change":"2024-09-21T01:46:28.651690+0000","last_active":"2024-09-21T01:46:32.064136+0000","last_peered":"2024-09-21T01:46:32.064136+0000","last_clean":"2024-09-21T01:46:32.064136+0000","last_became_active":"2024-09-21T01:46:28.651383+0000","last_became_peered":"2024-09-21T01:46:28.651383+0000","last_unstale":"2024-09-21T01:46:32.064136+0000","last_undegraded":"2024-09-21T01:46:32.064136+0000","last_fullsized":"2024-09-21T01:46:32.064136+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T01:49:28.532164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040868700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":39,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657657+0000","last_change":"2024-09-21T01:46:26.623500+0000","last_active":"2024-09-21T01:46:31.657657+0000","last_peered":"2024-09-21T01:46:31.657657+0000","last_clean":"2024-09-21T01:46:31.657657+0000","last_became_active":"2024-09-21T01:46:26.623142+0000","last_became_peered":"2024-09-21T01:46:26.623142+0000","last_unstale":"2024-09-21T01:46:31.657657+0000","last_undegraded":"2024-09-21T01:46:31.657657+0000","last_fullsized":"2024-09-21T01:46:31.657657+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:05:55.215381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055542700000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":49,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064256+0000","last_change":"2024-09-21T01:46:28.649216+0000","last_active":"2024-09-21T01:46:32.064256+0000","last_peered":"2024-09-21T01:46:32.064256+0000","last_clean":"2024-09-21T01:46:32.064256+0000","last_became_active":"2024-09-21T01:46:28.648886+0000","last_became_peered":"2024-09-21T01:46:28.648886+0000","last_unstale":"2024-09-21T01:46:32.064256+0000","last_undegraded":"2024-09-21T01:46:32.064256+0000","last_fullsized":"2024-09-21T01:46:32.064256+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:09:06.681843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00119484,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":39,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657839+0000","last_change":"2024-09-21T01:46:26.624399+0000","last_active":"2024-09-21T01:46:31.657839+0000","last_peered":"2024-09-21T01:46:31.657839+0000","last_clean":"2024-09-21T01:46:31.657839+0000","last_became_active":"2024-09-21T01:46:26.624073+0000","last_became_peered":"2024-09-21T01:46:26.624073+0000","last_unstale":"2024-09-21T01:46:31.657839+0000","last_undegraded":"2024-09-21T01:46:31.657839+0000","last_fullsized":"2024-09-21T01:46:31.657839+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T08:18:28.384798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063215099999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":46,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064137+0000","last_change":"2024-09-21T01:46:28.654323+0000","last_active":"2024-09-21T01:46:32.064137+0000","last_peered":"2024-09-21T01:46:32.064137+0000","last_clean":"2024-09-21T01:46:32.064137+0000","last_became_active":"2024-09-21T01:46:28.653984+0000","last_became_peered":"2024-09-21T01:46:28.653984+0000","last_unstale":"2024-09-21T01:46:32.064137+0000","last_undegraded":"2024-09-21T01:46:32.064137+0000","last_fullsized":"2024-09-21T01:46:32.064137+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:41:46.114559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010245149999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":36,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657473+0000","last_change":"2024-09-21T01:46:28.656954+0000","last_active":"2024-09-21T01:46:31.657473+0000","last_peered":"2024-09-21T01:46:31.657473+0000","last_clean":"2024-09-21T01:46:31.657473+0000","last_became_active":"2024-09-21T01:46:28.656588+0000","last_became_peered":"2024-09-21T01:46:28.656588+0000","last_unstale":"2024-09-21T01:46:31.657473+0000","last_undegraded":"2024-09-21T01:46:31.657473+0000","last_fullsized":"2024-09-21T01:46:31.657473+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T02:10:00.211563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080666500000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":44,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064104+0000","last_change":"2024-09-21T01:46:28.657038+0000","last_active":"2024-09-21T01:46:32.064104+0000","last_peered":"2024-09-21T01:46:32.064104+0000","last_clean":"2024-09-21T01:46:32.064104+0000","last_became_active":"2024-09-21T01:46:28.656701+0000","last_became_peered":"2024-09-21T01:46:28.656701+0000","last_unstale":"2024-09-21T01:46:32.064104+0000","last_undegraded":"2024-09-21T01:46:32.064104+0000","last_fullsized":"2024-09-21T01:46:32.064104+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T05:11:34.297379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058759600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":42,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064133+0000","last_change":"2024-09-21T01:46:28.657189+0000","last_active":"2024-09-21T01:46:32.064133+0000","last_peered":"2024-09-21T01:46:32.064133+0000","last_clean":"2024-09-21T01:46:32.064133+0000","last_became_active":"2024-09-21T01:46:28.656728+0000","last_became_peered":"2024-09-21T01:46:28.656728+0000","last_unstale":"2024-09-21T01:46:32.064133+0000","last_undegraded":"2024-09-21T01:46:32.064133+0000","last_fullsized":"2024-09-21T01:46:32.064133+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T08:49:15.772867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036256899999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"20'104","reported_seq":189,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657393+0000","last_change":"2024-09-21T01:46:26.626044+0000","last_active":"2024-09-21T01:46:31.657393+0000","last_peered":"2024-09-21T01:46:31.657393+0000","last_clean":"2024-09-21T01:46:31.657393+0000","last_became_active":"2024-09-21T01:46:26.625434+0000","last_became_peered":"2024-09-21T01:46:26.625434+0000","last_unstale":"2024-09-21T01:46:31.657393+0000","last_undegraded":"2024-09-21T01:46:31.657393+0000","last_fullsized":"2024-09-21T01:46:31.657393+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:50.489745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:50.489745+0000","last_clean_scrub_stamp":"2024-09-21T01:44:50.489745+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T08:39:55.782994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":29,"seq":124554051587,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27816,"kb_used_data":276,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344024,"statfs":{"total":96636764160,"available":96608280576,"internally_reserved":0,"allocated":282624,"data_stored":97230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4435,"internal_metadata":28176045},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":29,"seq":124554051587,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27812,"kb_used_data":272,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344028,"statfs":{"total":96636764160,"available":96608284672,"internally_reserved":0,"allocated":278528,"data_stored":97211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28176043},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":27,"seq":115964116996,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28328,"kb_used_data":856,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343512,"statfs":{"total":96636764160,"available":96607756288,"internally_reserved":0,"allocated":876544,"data_stored":687598,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4436,"internal_metadata":28110508},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":26,"seq":111669149700,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28392,"kb_used_data":852,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94343448,"statfs":{"total":96636764160,"available":96607690752,"internally_reserved":0,"allocated":872448,"data_stored":687579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28176043},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T01:46:35.967 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T01:46:36.078 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.077+0000 7f5b08314640 1 -- 172.21.15.105:0/3879572300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b00132cc0 msgr2=0x7f5b001350e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:36.078 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.077+0000 7f5b08314640 1 --2- 172.21.15.105:0/3879572300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b00132cc0 0x7f5b001350e0 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f5afc009830 tx=0x7f5afc02a480 comp rx=0 tx=0).stop 2024-09-21T01:46:36.078 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.077+0000 7f5b08314640 1 -- 172.21.15.105:0/3879572300 shutdown_connections 2024-09-21T01:46:36.078 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.077+0000 7f5b08314640 1 --2- 172.21.15.105:0/3879572300 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b00132cc0 0x7f5b001350e0 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.078 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.077+0000 7f5b08314640 1 --2- 172.21.15.105:0/3879572300 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5b00094b40 0x7f5b00132780 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.078 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.077+0000 7f5b08314640 1 -- 172.21.15.105:0/3879572300 >> 172.21.15.105:0/3879572300 conn(0x7f5b0009e240 msgr2=0x7f5b0009a710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:36.078 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.077+0000 7f5b08314640 1 -- 172.21.15.105:0/3879572300 shutdown_connections 2024-09-21T01:46:36.078 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.077+0000 7f5b08314640 1 -- 172.21.15.105:0/3879572300 wait complete. 2024-09-21T01:46:36.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.078+0000 7f5b08314640 1 Processor -- start 2024-09-21T01:46:36.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.078+0000 7f5b08314640 1 -- start start 2024-09-21T01:46:36.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.078+0000 7f5b08314640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b00094b40 0x7f5b001c3930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:36.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.078+0000 7f5b08314640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f5b00097da0 0x7f5b001c3e70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:36.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.078+0000 7f5b08314640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5b00132cc0 0x7f5b001c43b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:36.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.078+0000 7f5b08314640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f5b001364e0 con 0x7f5b00094b40 2024-09-21T01:46:36.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.078+0000 7f5b08314640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f5b00136360 con 0x7f5b00097da0 2024-09-21T01:46:36.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.078+0000 7f5b08314640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f5b00136660 con 0x7f5b00132cc0 2024-09-21T01:46:36.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.079+0000 7f5b068b1640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5b00132cc0 0x7f5b001c43b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:36.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.079+0000 7f5b060b0640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b00094b40 0x7f5b001c3930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:36.082 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.079+0000 7f5b058af640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f5b00097da0 0x7f5b001c3e70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:36.082 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.079+0000 7f5b068b1640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5b00132cc0 0x7f5b001c43b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:48582/0 (socket says 172.21.15.105:48582) 2024-09-21T01:46:36.082 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.079+0000 7f5b068b1640 1 -- 172.21.15.105:0/3073454655 learned_addr learned my addr 172.21.15.105:0/3073454655 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:36.082 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.079+0000 7f5b060b0640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b00094b40 0x7f5b001c3930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:52280/0 (socket says 172.21.15.105:52280) 2024-09-21T01:46:36.082 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.079+0000 7f5b068b1640 1 -- 172.21.15.105:0/3073454655 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f5b00097da0 msgr2=0x7f5b001c3e70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:36.082 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.079+0000 7f5b068b1640 1 --2- 172.21.15.105:0/3073454655 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f5b00097da0 0x7f5b001c3e70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.082 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.079+0000 7f5b068b1640 1 -- 172.21.15.105:0/3073454655 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b00094b40 msgr2=0x7f5b001c3930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:36.082 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.079+0000 7f5b068b1640 1 --2- 172.21.15.105:0/3073454655 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b00094b40 0x7f5b001c3930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.082 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.079+0000 7f5b068b1640 1 -- 172.21.15.105:0/3073454655 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f5b00126570 con 0x7f5b00132cc0 2024-09-21T01:46:36.082 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.079+0000 7f5b058af640 1 --2- 172.21.15.105:0/3073454655 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f5b00097da0 0x7f5b001c3e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:36.082 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.079+0000 7f5b060b0640 1 --2- 172.21.15.105:0/3073454655 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b00094b40 0x7f5b001c3930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:36.082 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.079+0000 7f5b068b1640 1 --2- 172.21.15.105:0/3073454655 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5b00132cc0 0x7f5b001c43b0 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f5afc02aa70 tx=0x7f5afc033c20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:36.082 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.079+0000 7f5af6ffd640 1 -- 172.21.15.105:0/3073454655 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5afc005c90 con 0x7f5b00132cc0 2024-09-21T01:46:36.083 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.079+0000 7f5af6ffd640 1 -- 172.21.15.105:0/3073454655 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f5afc031e50 con 0x7f5b00132cc0 2024-09-21T01:46:36.083 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.079+0000 7f5b08314640 1 -- 172.21.15.105:0/3073454655 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f5b00127520 con 0x7f5b00132cc0 2024-09-21T01:46:36.083 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.079+0000 7f5af6ffd640 1 -- 172.21.15.105:0/3073454655 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f5afc005220 con 0x7f5b00132cc0 2024-09-21T01:46:36.083 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.079+0000 7f5b08314640 1 -- 172.21.15.105:0/3073454655 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f5b00124dc0 con 0x7f5b00132cc0 2024-09-21T01:46:36.083 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.080+0000 7f5b08314640 1 -- 172.21.15.105:0/3073454655 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f5b00094430 con 0x7f5b00132cc0 2024-09-21T01:46:36.083 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.081+0000 7f5af6ffd640 1 -- 172.21.15.105:0/3073454655 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f5afc0053c0 con 0x7f5b00132cc0 2024-09-21T01:46:36.083 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.081+0000 7f5af6ffd640 1 -- 172.21.15.105:0/3073454655 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f5afc089fd0 con 0x7f5b00132cc0 2024-09-21T01:46:36.085 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.085+0000 7f5af6ffd640 1 -- 172.21.15.105:0/3073454655 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f5afc03f030 con 0x7f5b00132cc0 2024-09-21T01:46:36.230 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.230+0000 7f5b08314640 1 -- 172.21.15.105:0/3073454655 --> v1:172.21.15.105:6816/2838005468 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f5b00125d70 con 0x7f5ae40514b0 2024-09-21T01:46:36.231 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.231+0000 7f5af6ffd640 1 -- 172.21.15.105:0/3073454655 <== mgr.14125 v1:172.21.15.105:6816/2838005468 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32001 (unknown 2965378022 0 4261581420) 0x7f5b00125d70 con 0x7f5ae40514b0 2024-09-21T01:46:36.232 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:36.232 INFO:teuthology.orchestra.run.smithi105.stderr:dumped all 2024-09-21T01:46:36.233 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.233+0000 7f5b08314640 1 -- 172.21.15.105:0/3073454655 >> v1:172.21.15.105:6816/2838005468 conn(0x7f5ae40514b0 legacy=0x7f5ae40539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:36.234 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.233+0000 7f5b08314640 1 -- 172.21.15.105:0/3073454655 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5b00132cc0 msgr2=0x7f5b001c43b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:36.234 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.233+0000 7f5b08314640 1 --2- 172.21.15.105:0/3073454655 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5b00132cc0 0x7f5b001c43b0 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f5afc02aa70 tx=0x7f5afc033c20 comp rx=0 tx=0).stop 2024-09-21T01:46:36.234 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.233+0000 7f5b08314640 1 -- 172.21.15.105:0/3073454655 shutdown_connections 2024-09-21T01:46:36.234 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.233+0000 7f5b08314640 1 --2- 172.21.15.105:0/3073454655 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f5b00132cc0 0x7f5b001c43b0 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.234 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.233+0000 7f5b08314640 1 --2- 172.21.15.105:0/3073454655 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f5b00097da0 0x7f5b001c3e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.234 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.233+0000 7f5b08314640 1 --2- 172.21.15.105:0/3073454655 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f5b00094b40 0x7f5b001c3930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.234 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.233+0000 7f5b08314640 1 -- 172.21.15.105:0/3073454655 >> 172.21.15.105:0/3073454655 conn(0x7f5b0009e240 msgr2=0x7f5b00133a90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:36.234 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.233+0000 7f5b08314640 1 -- 172.21.15.105:0/3073454655 shutdown_connections 2024-09-21T01:46:36.234 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.233+0000 7f5b08314640 1 -- 172.21.15.105:0/3073454655 wait complete. 2024-09-21T01:46:36.244 INFO:teuthology.orchestra.run.smithi105.stdout:{"pg_ready":true,"pg_map":{"version":27,"stamp":"2024-09-21T01:46:35.501509+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112348,"kb_used_data":2256,"kb_used_omap":17,"kb_used_meta":109998,"kb_avail":377375012,"statfs":{"total":386547056640,"available":386432012288,"internally_reserved":0,"allocated":2310144,"data_stored":1569618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17745,"internal_metadata":112638639},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001190"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":49,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064136+0000","last_change":"2024-09-21T01:46:28.651690+0000","last_active":"2024-09-21T01:46:32.064136+0000","last_peered":"2024-09-21T01:46:32.064136+0000","last_clean":"2024-09-21T01:46:32.064136+0000","last_became_active":"2024-09-21T01:46:28.651383+0000","last_became_peered":"2024-09-21T01:46:28.651383+0000","last_unstale":"2024-09-21T01:46:32.064136+0000","last_undegraded":"2024-09-21T01:46:32.064136+0000","last_fullsized":"2024-09-21T01:46:32.064136+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T01:49:28.532164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040868700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":39,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657657+0000","last_change":"2024-09-21T01:46:26.623500+0000","last_active":"2024-09-21T01:46:31.657657+0000","last_peered":"2024-09-21T01:46:31.657657+0000","last_clean":"2024-09-21T01:46:31.657657+0000","last_became_active":"2024-09-21T01:46:26.623142+0000","last_became_peered":"2024-09-21T01:46:26.623142+0000","last_unstale":"2024-09-21T01:46:31.657657+0000","last_undegraded":"2024-09-21T01:46:31.657657+0000","last_fullsized":"2024-09-21T01:46:31.657657+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:05:55.215381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055542700000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":49,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064256+0000","last_change":"2024-09-21T01:46:28.649216+0000","last_active":"2024-09-21T01:46:32.064256+0000","last_peered":"2024-09-21T01:46:32.064256+0000","last_clean":"2024-09-21T01:46:32.064256+0000","last_became_active":"2024-09-21T01:46:28.648886+0000","last_became_peered":"2024-09-21T01:46:28.648886+0000","last_unstale":"2024-09-21T01:46:32.064256+0000","last_undegraded":"2024-09-21T01:46:32.064256+0000","last_fullsized":"2024-09-21T01:46:32.064256+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:09:06.681843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00119484,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":39,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657839+0000","last_change":"2024-09-21T01:46:26.624399+0000","last_active":"2024-09-21T01:46:31.657839+0000","last_peered":"2024-09-21T01:46:31.657839+0000","last_clean":"2024-09-21T01:46:31.657839+0000","last_became_active":"2024-09-21T01:46:26.624073+0000","last_became_peered":"2024-09-21T01:46:26.624073+0000","last_unstale":"2024-09-21T01:46:31.657839+0000","last_undegraded":"2024-09-21T01:46:31.657839+0000","last_fullsized":"2024-09-21T01:46:31.657839+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T08:18:28.384798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063215099999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":46,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064137+0000","last_change":"2024-09-21T01:46:28.654323+0000","last_active":"2024-09-21T01:46:32.064137+0000","last_peered":"2024-09-21T01:46:32.064137+0000","last_clean":"2024-09-21T01:46:32.064137+0000","last_became_active":"2024-09-21T01:46:28.653984+0000","last_became_peered":"2024-09-21T01:46:28.653984+0000","last_unstale":"2024-09-21T01:46:32.064137+0000","last_undegraded":"2024-09-21T01:46:32.064137+0000","last_fullsized":"2024-09-21T01:46:32.064137+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:41:46.114559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010245149999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":36,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657473+0000","last_change":"2024-09-21T01:46:28.656954+0000","last_active":"2024-09-21T01:46:31.657473+0000","last_peered":"2024-09-21T01:46:31.657473+0000","last_clean":"2024-09-21T01:46:31.657473+0000","last_became_active":"2024-09-21T01:46:28.656588+0000","last_became_peered":"2024-09-21T01:46:28.656588+0000","last_unstale":"2024-09-21T01:46:31.657473+0000","last_undegraded":"2024-09-21T01:46:31.657473+0000","last_fullsized":"2024-09-21T01:46:31.657473+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T02:10:00.211563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080666500000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":44,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064104+0000","last_change":"2024-09-21T01:46:28.657038+0000","last_active":"2024-09-21T01:46:32.064104+0000","last_peered":"2024-09-21T01:46:32.064104+0000","last_clean":"2024-09-21T01:46:32.064104+0000","last_became_active":"2024-09-21T01:46:28.656701+0000","last_became_peered":"2024-09-21T01:46:28.656701+0000","last_unstale":"2024-09-21T01:46:32.064104+0000","last_undegraded":"2024-09-21T01:46:32.064104+0000","last_fullsized":"2024-09-21T01:46:32.064104+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T05:11:34.297379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058759600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":42,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064133+0000","last_change":"2024-09-21T01:46:28.657189+0000","last_active":"2024-09-21T01:46:32.064133+0000","last_peered":"2024-09-21T01:46:32.064133+0000","last_clean":"2024-09-21T01:46:32.064133+0000","last_became_active":"2024-09-21T01:46:28.656728+0000","last_became_peered":"2024-09-21T01:46:28.656728+0000","last_unstale":"2024-09-21T01:46:32.064133+0000","last_undegraded":"2024-09-21T01:46:32.064133+0000","last_fullsized":"2024-09-21T01:46:32.064133+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T08:49:15.772867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036256899999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"20'104","reported_seq":189,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657393+0000","last_change":"2024-09-21T01:46:26.626044+0000","last_active":"2024-09-21T01:46:31.657393+0000","last_peered":"2024-09-21T01:46:31.657393+0000","last_clean":"2024-09-21T01:46:31.657393+0000","last_became_active":"2024-09-21T01:46:26.625434+0000","last_became_peered":"2024-09-21T01:46:26.625434+0000","last_unstale":"2024-09-21T01:46:31.657393+0000","last_undegraded":"2024-09-21T01:46:31.657393+0000","last_fullsized":"2024-09-21T01:46:31.657393+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:50.489745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:50.489745+0000","last_clean_scrub_stamp":"2024-09-21T01:44:50.489745+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T08:39:55.782994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":29,"seq":124554051587,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27816,"kb_used_data":276,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344024,"statfs":{"total":96636764160,"available":96608280576,"internally_reserved":0,"allocated":282624,"data_stored":97230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4435,"internal_metadata":28176045},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":29,"seq":124554051587,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27812,"kb_used_data":272,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344028,"statfs":{"total":96636764160,"available":96608284672,"internally_reserved":0,"allocated":278528,"data_stored":97211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28176043},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":27,"seq":115964116996,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28328,"kb_used_data":856,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343512,"statfs":{"total":96636764160,"available":96607756288,"internally_reserved":0,"allocated":876544,"data_stored":687598,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4436,"internal_metadata":28110508},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":26,"seq":111669149700,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28392,"kb_used_data":852,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94343448,"statfs":{"total":96636764160,"available":96607690752,"internally_reserved":0,"allocated":872448,"data_stored":687579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28176043},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T01:46:36.245 INFO:tasks.ceph.ceph_manager.ceph:clean! 2024-09-21T01:46:36.245 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T01:46:36.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.355+0000 7f4722de4640 1 -- 172.21.15.105:0/467110072 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f471c0733e0 msgr2=0x7f471c0737f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:36.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.355+0000 7f4722de4640 1 --2- 172.21.15.105:0/467110072 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f471c0733e0 0x7f471c0737f0 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7f4718005fd0 tx=0x7f471802a4a0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.356 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.355+0000 7f4722de4640 1 -- 172.21.15.105:0/467110072 shutdown_connections 2024-09-21T01:46:36.356 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.355+0000 7f4722de4640 1 --2- 172.21.15.105:0/467110072 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f471c12c440 0x7f471c132e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.356 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.355+0000 7f4722de4640 1 --2- 172.21.15.105:0/467110072 >> v2:172.21.15.105:3300/0 conn(0x7f471c12baf0 0x7f471c12bf00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.356 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.355+0000 7f4722de4640 1 --2- 172.21.15.105:0/467110072 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f471c0733e0 0x7f471c0737f0 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.356 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.355+0000 7f4722de4640 1 -- 172.21.15.105:0/467110072 >> 172.21.15.105:0/467110072 conn(0x7f471c09b7c0 msgr2=0x7f471c09bbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:36.356 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.355+0000 7f4722de4640 1 -- 172.21.15.105:0/467110072 shutdown_connections 2024-09-21T01:46:36.356 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.355+0000 7f4722de4640 1 -- 172.21.15.105:0/467110072 wait complete. 2024-09-21T01:46:36.358 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.356+0000 7f4722de4640 1 Processor -- start 2024-09-21T01:46:36.358 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.356+0000 7f4722de4640 1 -- start start 2024-09-21T01:46:36.359 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f4722de4640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f471c0733e0 0x7f471c127740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:36.359 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f4722de4640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f471c12baf0 0x7f471c127c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:36.359 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f4722de4640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f471c12c440 0x7f471c1281c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:36.359 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f4722de4640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f471c133920 con 0x7f471c0733e0 2024-09-21T01:46:36.359 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f4722de4640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f471c1337a0 con 0x7f471c12c440 2024-09-21T01:46:36.359 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f4722de4640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f471c133aa0 con 0x7f471c12baf0 2024-09-21T01:46:36.359 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f4713fff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f471c12baf0 0x7f471c127c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:36.359 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f4720b80640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f471c0733e0 0x7f471c127740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:36.359 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f4713fff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f471c12baf0 0x7f471c127c80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:48602/0 (socket says 172.21.15.105:48602) 2024-09-21T01:46:36.359 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f4720b80640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f471c0733e0 0x7f471c127740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:52302/0 (socket says 172.21.15.105:52302) 2024-09-21T01:46:36.359 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f4721381640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f471c12c440 0x7f471c1281c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:36.359 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f4713fff640 1 -- 172.21.15.105:0/2024850579 learned_addr learned my addr 172.21.15.105:0/2024850579 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:36.359 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f4720b80640 1 -- 172.21.15.105:0/2024850579 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f471c12baf0 msgr2=0x7f471c127c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:36.360 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f4720b80640 1 --2- 172.21.15.105:0/2024850579 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f471c12baf0 0x7f471c127c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.360 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f4720b80640 1 -- 172.21.15.105:0/2024850579 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f471c12c440 msgr2=0x7f471c1281c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:36.360 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f4720b80640 1 --2- 172.21.15.105:0/2024850579 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f471c12c440 0x7f471c1281c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.360 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f4720b80640 1 -- 172.21.15.105:0/2024850579 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f471c12aa90 con 0x7f471c0733e0 2024-09-21T01:46:36.360 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f4713fff640 1 --2- 172.21.15.105:0/2024850579 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f471c12baf0 0x7f471c127c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:36.360 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f4721381640 1 --2- 172.21.15.105:0/2024850579 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f471c12c440 0x7f471c1281c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:36.360 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f4720b80640 1 --2- 172.21.15.105:0/2024850579 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f471c0733e0 0x7f471c127740 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f4718031000 tx=0x7f471803b930 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:36.360 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f47117fa640 1 -- 172.21.15.105:0/2024850579 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4718048070 con 0x7f471c0733e0 2024-09-21T01:46:36.360 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f47117fa640 1 -- 172.21.15.105:0/2024850579 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f47180035f0 con 0x7f471c0733e0 2024-09-21T01:46:36.360 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.357+0000 7f4722de4640 1 -- 172.21.15.105:0/2024850579 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f471c12ad50 con 0x7f471c0733e0 2024-09-21T01:46:36.360 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.358+0000 7f47117fa640 1 -- 172.21.15.105:0/2024850579 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f4718035040 con 0x7f471c0733e0 2024-09-21T01:46:36.360 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.358+0000 7f4722de4640 1 -- 172.21.15.105:0/2024850579 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7f471c12cb00 con 0x7f471c0733e0 2024-09-21T01:46:36.361 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.359+0000 7f47117fa640 1 -- 172.21.15.105:0/2024850579 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f471804b070 con 0x7f471c0733e0 2024-09-21T01:46:36.361 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.359+0000 7f4722de4640 1 -- 172.21.15.105:0/2024850579 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f471c070730 con 0x7f471c0733e0 2024-09-21T01:46:36.361 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.359+0000 7f47117fa640 1 -- 172.21.15.105:0/2024850579 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f471808d220 con 0x7f471c0733e0 2024-09-21T01:46:36.363 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.363+0000 7f47117fa640 1 -- 172.21.15.105:0/2024850579 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f471c070730 con 0x7f471c0733e0 2024-09-21T01:46:36.507 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.506+0000 7f4722de4640 1 -- 172.21.15.105:0/2024850579 --> v1:172.21.15.105:6816/2838005468 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f471c129d60 con 0x7f47080514b0 2024-09-21T01:46:36.507 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.507+0000 7f47117fa640 1 -- 172.21.15.105:0/2024850579 <== mgr.14125 v1:172.21.15.105:6816/2838005468 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32001 (unknown 2965378022 0 4261581420) 0x7f471c129d60 con 0x7f47080514b0 2024-09-21T01:46:36.508 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:36.508 INFO:teuthology.orchestra.run.smithi105.stderr:dumped all 2024-09-21T01:46:36.510 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.509+0000 7f4722de4640 1 -- 172.21.15.105:0/2024850579 >> v1:172.21.15.105:6816/2838005468 conn(0x7f47080514b0 legacy=0x7f47080539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:36.510 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.509+0000 7f4722de4640 1 -- 172.21.15.105:0/2024850579 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f471c0733e0 msgr2=0x7f471c127740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:36.510 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.509+0000 7f4722de4640 1 --2- 172.21.15.105:0/2024850579 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f471c0733e0 0x7f471c127740 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f4718031000 tx=0x7f471803b930 comp rx=0 tx=0).stop 2024-09-21T01:46:36.510 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.509+0000 7f4722de4640 1 -- 172.21.15.105:0/2024850579 shutdown_connections 2024-09-21T01:46:36.510 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.509+0000 7f4722de4640 1 --2- 172.21.15.105:0/2024850579 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f471c12c440 0x7f471c1281c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.510 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.509+0000 7f4722de4640 1 --2- 172.21.15.105:0/2024850579 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f471c12baf0 0x7f471c127c80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.510 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.509+0000 7f4722de4640 1 --2- 172.21.15.105:0/2024850579 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f471c0733e0 0x7f471c127740 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.510 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.509+0000 7f4722de4640 1 -- 172.21.15.105:0/2024850579 >> 172.21.15.105:0/2024850579 conn(0x7f471c09b7c0 msgr2=0x7f471c0972b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:36.510 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.509+0000 7f4722de4640 1 -- 172.21.15.105:0/2024850579 shutdown_connections 2024-09-21T01:46:36.510 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.509+0000 7f4722de4640 1 -- 172.21.15.105:0/2024850579 wait complete. 2024-09-21T01:46:36.521 INFO:teuthology.orchestra.run.smithi105.stdout:{"pg_ready":true,"pg_map":{"version":27,"stamp":"2024-09-21T01:46:35.501509+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112348,"kb_used_data":2256,"kb_used_omap":17,"kb_used_meta":109998,"kb_avail":377375012,"statfs":{"total":386547056640,"available":386432012288,"internally_reserved":0,"allocated":2310144,"data_stored":1569618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17745,"internal_metadata":112638639},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001190"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":49,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064136+0000","last_change":"2024-09-21T01:46:28.651690+0000","last_active":"2024-09-21T01:46:32.064136+0000","last_peered":"2024-09-21T01:46:32.064136+0000","last_clean":"2024-09-21T01:46:32.064136+0000","last_became_active":"2024-09-21T01:46:28.651383+0000","last_became_peered":"2024-09-21T01:46:28.651383+0000","last_unstale":"2024-09-21T01:46:32.064136+0000","last_undegraded":"2024-09-21T01:46:32.064136+0000","last_fullsized":"2024-09-21T01:46:32.064136+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T01:49:28.532164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040868700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":39,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657657+0000","last_change":"2024-09-21T01:46:26.623500+0000","last_active":"2024-09-21T01:46:31.657657+0000","last_peered":"2024-09-21T01:46:31.657657+0000","last_clean":"2024-09-21T01:46:31.657657+0000","last_became_active":"2024-09-21T01:46:26.623142+0000","last_became_peered":"2024-09-21T01:46:26.623142+0000","last_unstale":"2024-09-21T01:46:31.657657+0000","last_undegraded":"2024-09-21T01:46:31.657657+0000","last_fullsized":"2024-09-21T01:46:31.657657+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:05:55.215381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055542700000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":49,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064256+0000","last_change":"2024-09-21T01:46:28.649216+0000","last_active":"2024-09-21T01:46:32.064256+0000","last_peered":"2024-09-21T01:46:32.064256+0000","last_clean":"2024-09-21T01:46:32.064256+0000","last_became_active":"2024-09-21T01:46:28.648886+0000","last_became_peered":"2024-09-21T01:46:28.648886+0000","last_unstale":"2024-09-21T01:46:32.064256+0000","last_undegraded":"2024-09-21T01:46:32.064256+0000","last_fullsized":"2024-09-21T01:46:32.064256+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:09:06.681843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00119484,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":39,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657839+0000","last_change":"2024-09-21T01:46:26.624399+0000","last_active":"2024-09-21T01:46:31.657839+0000","last_peered":"2024-09-21T01:46:31.657839+0000","last_clean":"2024-09-21T01:46:31.657839+0000","last_became_active":"2024-09-21T01:46:26.624073+0000","last_became_peered":"2024-09-21T01:46:26.624073+0000","last_unstale":"2024-09-21T01:46:31.657839+0000","last_undegraded":"2024-09-21T01:46:31.657839+0000","last_fullsized":"2024-09-21T01:46:31.657839+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T08:18:28.384798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063215099999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":46,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064137+0000","last_change":"2024-09-21T01:46:28.654323+0000","last_active":"2024-09-21T01:46:32.064137+0000","last_peered":"2024-09-21T01:46:32.064137+0000","last_clean":"2024-09-21T01:46:32.064137+0000","last_became_active":"2024-09-21T01:46:28.653984+0000","last_became_peered":"2024-09-21T01:46:28.653984+0000","last_unstale":"2024-09-21T01:46:32.064137+0000","last_undegraded":"2024-09-21T01:46:32.064137+0000","last_fullsized":"2024-09-21T01:46:32.064137+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:41:46.114559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010245149999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":36,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657473+0000","last_change":"2024-09-21T01:46:28.656954+0000","last_active":"2024-09-21T01:46:31.657473+0000","last_peered":"2024-09-21T01:46:31.657473+0000","last_clean":"2024-09-21T01:46:31.657473+0000","last_became_active":"2024-09-21T01:46:28.656588+0000","last_became_peered":"2024-09-21T01:46:28.656588+0000","last_unstale":"2024-09-21T01:46:31.657473+0000","last_undegraded":"2024-09-21T01:46:31.657473+0000","last_fullsized":"2024-09-21T01:46:31.657473+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T02:10:00.211563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080666500000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":44,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064104+0000","last_change":"2024-09-21T01:46:28.657038+0000","last_active":"2024-09-21T01:46:32.064104+0000","last_peered":"2024-09-21T01:46:32.064104+0000","last_clean":"2024-09-21T01:46:32.064104+0000","last_became_active":"2024-09-21T01:46:28.656701+0000","last_became_peered":"2024-09-21T01:46:28.656701+0000","last_unstale":"2024-09-21T01:46:32.064104+0000","last_undegraded":"2024-09-21T01:46:32.064104+0000","last_fullsized":"2024-09-21T01:46:32.064104+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T05:11:34.297379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058759600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":42,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064133+0000","last_change":"2024-09-21T01:46:28.657189+0000","last_active":"2024-09-21T01:46:32.064133+0000","last_peered":"2024-09-21T01:46:32.064133+0000","last_clean":"2024-09-21T01:46:32.064133+0000","last_became_active":"2024-09-21T01:46:28.656728+0000","last_became_peered":"2024-09-21T01:46:28.656728+0000","last_unstale":"2024-09-21T01:46:32.064133+0000","last_undegraded":"2024-09-21T01:46:32.064133+0000","last_fullsized":"2024-09-21T01:46:32.064133+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T08:49:15.772867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036256899999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"20'104","reported_seq":189,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657393+0000","last_change":"2024-09-21T01:46:26.626044+0000","last_active":"2024-09-21T01:46:31.657393+0000","last_peered":"2024-09-21T01:46:31.657393+0000","last_clean":"2024-09-21T01:46:31.657393+0000","last_became_active":"2024-09-21T01:46:26.625434+0000","last_became_peered":"2024-09-21T01:46:26.625434+0000","last_unstale":"2024-09-21T01:46:31.657393+0000","last_undegraded":"2024-09-21T01:46:31.657393+0000","last_fullsized":"2024-09-21T01:46:31.657393+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:50.489745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:50.489745+0000","last_clean_scrub_stamp":"2024-09-21T01:44:50.489745+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T08:39:55.782994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":29,"seq":124554051587,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27816,"kb_used_data":276,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344024,"statfs":{"total":96636764160,"available":96608280576,"internally_reserved":0,"allocated":282624,"data_stored":97230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4435,"internal_metadata":28176045},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":29,"seq":124554051587,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27812,"kb_used_data":272,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344028,"statfs":{"total":96636764160,"available":96608284672,"internally_reserved":0,"allocated":278528,"data_stored":97211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28176043},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":27,"seq":115964116996,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28328,"kb_used_data":856,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343512,"statfs":{"total":96636764160,"available":96607756288,"internally_reserved":0,"allocated":876544,"data_stored":687598,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4436,"internal_metadata":28110508},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":26,"seq":111669149700,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28392,"kb_used_data":852,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94343448,"statfs":{"total":96636764160,"available":96607690752,"internally_reserved":0,"allocated":872448,"data_stored":687579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28176043},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T01:46:36.522 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-09-21T01:46:36.632 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.631+0000 7ff3ebf26640 1 -- 172.21.15.105:0/311070669 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3e4071f70 msgr2=0x7ff3e4134830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:36.632 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.631+0000 7ff3ebf26640 1 --2- 172.21.15.105:0/311070669 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3e4071f70 0x7ff3e4134830 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7ff3dc005fd0 tx=0x7ff3dc02a780 comp rx=0 tx=0).stop 2024-09-21T01:46:36.632 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.631+0000 7ff3ebf26640 1 -- 172.21.15.105:0/311070669 shutdown_connections 2024-09-21T01:46:36.632 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.631+0000 7ff3ebf26640 1 --2- 172.21.15.105:0/311070669 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3e4071f70 0x7ff3e4134830 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.632 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.631+0000 7ff3ebf26640 1 --2- 172.21.15.105:0/311070669 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff3e409bce0 0x7ff3e412c8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.632 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.631+0000 7ff3ebf26640 1 -- 172.21.15.105:0/311070669 >> 172.21.15.105:0/311070669 conn(0x7ff3e409f000 msgr2=0x7ff3e409b4d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:36.632 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.631+0000 7ff3ebf26640 1 -- 172.21.15.105:0/311070669 shutdown_connections 2024-09-21T01:46:36.632 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.631+0000 7ff3ebf26640 1 -- 172.21.15.105:0/311070669 wait complete. 2024-09-21T01:46:36.635 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.632+0000 7ff3ebf26640 1 Processor -- start 2024-09-21T01:46:36.635 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.632+0000 7ff3ebf26640 1 -- start start 2024-09-21T01:46:36.635 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.633+0000 7ff3ebf26640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3e4071f70 0x7ff3e40980b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:36.635 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.633+0000 7ff3ebf26640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff3e409bce0 0x7ff3e40985f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:36.635 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.633+0000 7ff3ebf26640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff3e412cde0 0x7ff3e4098b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:36.635 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.633+0000 7ff3ebf26640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7ff3e4135490 con 0x7ff3e4071f70 2024-09-21T01:46:36.635 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.633+0000 7ff3ebf26640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7ff3e4135310 con 0x7ff3e412cde0 2024-09-21T01:46:36.635 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.633+0000 7ff3ebf26640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7ff3e4135610 con 0x7ff3e409bce0 2024-09-21T01:46:36.636 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.633+0000 7ff3e9cc2640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3e4071f70 0x7ff3e40980b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:36.636 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.633+0000 7ff3e94c1640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff3e409bce0 0x7ff3e40985f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:36.636 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.633+0000 7ff3e9cc2640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3e4071f70 0x7ff3e40980b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:52326/0 (socket says 172.21.15.105:52326) 2024-09-21T01:46:36.636 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.633+0000 7ff3e9cc2640 1 -- 172.21.15.105:0/589960223 learned_addr learned my addr 172.21.15.105:0/589960223 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:36.636 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.633+0000 7ff3e94c1640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff3e409bce0 0x7ff3e40985f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:48626/0 (socket says 172.21.15.105:48626) 2024-09-21T01:46:36.636 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.633+0000 7ff3ea4c3640 1 --2- 172.21.15.105:0/589960223 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff3e412cde0 0x7ff3e4098b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:36.636 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.633+0000 7ff3e94c1640 1 -- 172.21.15.105:0/589960223 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff3e412cde0 msgr2=0x7ff3e4098b30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:36.636 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.633+0000 7ff3e94c1640 1 --2- 172.21.15.105:0/589960223 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff3e412cde0 0x7ff3e4098b30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.636 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.633+0000 7ff3e94c1640 1 -- 172.21.15.105:0/589960223 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3e4071f70 msgr2=0x7ff3e40980b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:36.636 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.633+0000 7ff3e94c1640 1 --2- 172.21.15.105:0/589960223 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3e4071f70 0x7ff3e40980b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.636 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.633+0000 7ff3e94c1640 1 -- 172.21.15.105:0/589960223 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff3e4131fd0 con 0x7ff3e409bce0 2024-09-21T01:46:36.636 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.633+0000 7ff3e9cc2640 1 --2- 172.21.15.105:0/589960223 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3e4071f70 0x7ff3e40980b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:36.636 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.634+0000 7ff3e94c1640 1 --2- 172.21.15.105:0/589960223 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff3e409bce0 0x7ff3e40985f0 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7ff3d400c390 tx=0x7ff3d400c850 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:36.637 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.634+0000 7ff3da7fc640 1 -- 172.21.15.105:0/589960223 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff3d4017070 con 0x7ff3e409bce0 2024-09-21T01:46:36.637 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.634+0000 7ff3da7fc640 1 -- 172.21.15.105:0/589960223 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7ff3d40035f0 con 0x7ff3e409bce0 2024-09-21T01:46:36.637 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.634+0000 7ff3da7fc640 1 -- 172.21.15.105:0/589960223 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff3d4003a40 con 0x7ff3e409bce0 2024-09-21T01:46:36.637 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.634+0000 7ff3ebf26640 1 -- 172.21.15.105:0/589960223 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff3e406e9e0 con 0x7ff3e409bce0 2024-09-21T01:46:36.637 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.634+0000 7ff3ebf26640 1 -- 172.21.15.105:0/589960223 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7ff3e406ef80 con 0x7ff3e409bce0 2024-09-21T01:46:36.637 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.635+0000 7ff3ebf26640 1 -- 172.21.15.105:0/589960223 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff3e412c8a0 con 0x7ff3e409bce0 2024-09-21T01:46:36.637 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.635+0000 7ff3da7fc640 1 -- 172.21.15.105:0/589960223 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7ff3d401a070 con 0x7ff3e409bce0 2024-09-21T01:46:36.637 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.635+0000 7ff3da7fc640 1 -- 172.21.15.105:0/589960223 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7ff3d4068250 con 0x7ff3e409bce0 2024-09-21T01:46:36.639 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.639+0000 7ff3da7fc640 1 -- 172.21.15.105:0/589960223 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7ff3e412c490 con 0x7ff3e409bce0 2024-09-21T01:46:36.792 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.792+0000 7ff3ebf26640 1 -- 172.21.15.105:0/589960223 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) -- 0x7ff3e406a820 con 0x7ff3e409bce0 2024-09-21T01:46:36.793 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.793+0000 7ff3da7fc640 1 -- 172.21.15.105:0/589960223 <== mon.2 v2:172.21.15.105:3302/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v33) ==== 74+0+9196 (secure 0 0 0) 0x7ff3d4067300 con 0x7ff3e409bce0 2024-09-21T01:46:36.793 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:36.793 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":33,"fsid":"355399d3-6490-4ad0-ac52-d793648ff276","created":"2024-09-21T01:44:37.282024+0000","modified":"2024-09-21T01:46:31.651379+0000","last_up_change":"2024-09-21T01:46:27.619214+0000","last_in_change":"2024-09-21T01:44:39.666350+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":7,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":4,"require_min_compat_client":"reef","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T01:44:50.380956+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"32","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":4,"score_stable":4,"optimal_score":0.5,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-21T01:44:54.210449+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"33","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"03218bdb-ec48-44c7-bbf5-31cb53a3cea7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":20,"up_from":26,"up_thru":26,"down_at":21,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6800","nonce":249246859}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6801","nonce":249246859}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6803","nonce":249246859}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6802","nonce":249246859}]},"public_addr":"172.21.15.105:6800/249246859","cluster_addr":"172.21.15.105:6801/249246859","heartbeat_back_addr":"172.21.15.105:6803/249246859","heartbeat_front_addr":"172.21.15.105:6802/249246859","state":["exists","up"]},{"osd":1,"uuid":"5528ff44-922a-4e44-bb69-95fd9544d8dd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":20,"up_from":27,"up_thru":27,"down_at":22,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6804","nonce":1896987326}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6805","nonce":1896987326}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6807","nonce":1896987326}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6806","nonce":1896987326}]},"public_addr":"172.21.15.105:6804/1896987326","cluster_addr":"172.21.15.105:6805/1896987326","heartbeat_back_addr":"172.21.15.105:6807/1896987326","heartbeat_front_addr":"172.21.15.105:6806/1896987326","state":["exists","up"]},{"osd":2,"uuid":"df56ff35-71e9-4d46-8abb-84265eb6e1ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":22,"up_from":29,"up_thru":29,"down_at":23,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6808","nonce":2933198036}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6809","nonce":2933198036}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6811","nonce":2933198036}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6810","nonce":2933198036}]},"public_addr":"172.21.15.105:6808/2933198036","cluster_addr":"172.21.15.105:6809/2933198036","heartbeat_back_addr":"172.21.15.105:6811/2933198036","heartbeat_front_addr":"172.21.15.105:6810/2933198036","state":["exists","up"]},{"osd":3,"uuid":"012d9668-f1d3-4268-b0eb-15fa845ef17c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":9,"last_clean_end":23,"up_from":29,"up_thru":29,"down_at":24,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6812","nonce":568702146}]},"cluster_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6813","nonce":568702146}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6815","nonce":568702146}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v1","addr":"172.21.15.105:6814","nonce":568702146}]},"public_addr":"172.21.15.105:6812/568702146","cluster_addr":"172.21.15.105:6813/568702146","heartbeat_back_addr":"172.21.15.105:6815/568702146","heartbeat_front_addr":"172.21.15.105:6814/568702146","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.554795+0000","dead_epoch":21},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.869400+0000","dead_epoch":22},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.601687+0000","dead_epoch":23},{"osd":3,"down_stamp":"2024-09-21T01:46:21.001569+0000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T01:44:44.822680+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.105:0/3561680891":"2024-09-22T01:46:13.485080+0000","172.21.15.105:0/2570087590":"2024-09-22T01:46:13.485080+0000","172.21.15.105:0/3837276043":"2024-09-22T01:46:13.485080+0000","172.21.15.105:0/2810478014":"2024-09-22T01:46:13.485080+0000","172.21.15.105:6816/3933313069":"2024-09-22T01:46:13.485080+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T01:46:36.795 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.795+0000 7ff3ebf26640 1 -- 172.21.15.105:0/589960223 >> v1:172.21.15.105:6816/2838005468 conn(0x7ff3c80514b0 legacy=0x7ff3c80539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:36.796 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.795+0000 7ff3ebf26640 1 -- 172.21.15.105:0/589960223 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff3e409bce0 msgr2=0x7ff3e40985f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:36.796 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.795+0000 7ff3ebf26640 1 --2- 172.21.15.105:0/589960223 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff3e409bce0 0x7ff3e40985f0 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7ff3d400c390 tx=0x7ff3d400c850 comp rx=0 tx=0).stop 2024-09-21T01:46:36.796 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.795+0000 7ff3ebf26640 1 -- 172.21.15.105:0/589960223 shutdown_connections 2024-09-21T01:46:36.796 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.795+0000 7ff3ebf26640 1 --2- 172.21.15.105:0/589960223 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff3e412cde0 0x7ff3e4098b30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.796 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.795+0000 7ff3ebf26640 1 --2- 172.21.15.105:0/589960223 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff3e409bce0 0x7ff3e40985f0 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.796 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.795+0000 7ff3ebf26640 1 --2- 172.21.15.105:0/589960223 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff3e4071f70 0x7ff3e40980b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:36.796 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.795+0000 7ff3ebf26640 1 -- 172.21.15.105:0/589960223 >> 172.21.15.105:0/589960223 conn(0x7ff3e409f000 msgr2=0x7ff3e412bb90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:36.796 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.795+0000 7ff3ebf26640 1 -- 172.21.15.105:0/589960223 shutdown_connections 2024-09-21T01:46:36.796 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:36.795+0000 7ff3ebf26640 1 -- 172.21.15.105:0/589960223 wait complete. 2024-09-21T01:46:37.808 INFO:tasks.ceph:Scrubbing osd.0 2024-09-21T01:46:37.808 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 config set osd_debug_deep_scrub_sleep 0 2024-09-21T01:46:37.935 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.935+0000 7f81f4389640 1 -- 172.21.15.105:0/2924324022 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f81ec130370 msgr2=0x7f81ec0719c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:37.936 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.935+0000 7f81f4389640 1 --2- 172.21.15.105:0/2924324022 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f81ec130370 0x7f81ec0719c0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7f81dc005fd0 tx=0x7f81dc02a540 comp rx=0 tx=0).stop 2024-09-21T01:46:37.936 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.935+0000 7f81f4389640 1 -- 172.21.15.105:0/2924324022 shutdown_connections 2024-09-21T01:46:37.936 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.935+0000 7f81f4389640 1 --2- 172.21.15.105:0/2924324022 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f81ec134b20 0x7f81ec136f40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:37.936 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.935+0000 7f81f4389640 1 --2- 172.21.15.105:0/2924324022 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f81ec130370 0x7f81ec0719c0 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:37.936 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.935+0000 7f81f4389640 1 -- 172.21.15.105:0/2924324022 >> 172.21.15.105:0/2924324022 conn(0x7f81ec0a2260 msgr2=0x7f81ec0a2670 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:37.936 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.935+0000 7f81f4389640 1 -- 172.21.15.105:0/2924324022 shutdown_connections 2024-09-21T01:46:37.936 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.935+0000 7f81f4389640 1 -- 172.21.15.105:0/2924324022 wait complete. 2024-09-21T01:46:37.938 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.936+0000 7f81f4389640 1 Processor -- start 2024-09-21T01:46:37.938 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.936+0000 7f81f4389640 1 -- start start 2024-09-21T01:46:37.938 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.936+0000 7f81f4389640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f81ec130370 0x7f81ec12acd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:37.938 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.936+0000 7f81f4389640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f81ec134b20 0x7f81ec12b210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:37.938 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.936+0000 7f81f4389640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f81ec130a60 0x7f81ec12b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:37.938 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.936+0000 7f81f4389640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f81ec138960 con 0x7f81ec130a60 2024-09-21T01:46:37.939 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.936+0000 7f81f4389640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f81ec1381c0 con 0x7f81ec134b20 2024-09-21T01:46:37.939 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.936+0000 7f81f4389640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f81ec138ae0 con 0x7f81ec130370 2024-09-21T01:46:37.939 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.937+0000 7f81f2125640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f81ec130370 0x7f81ec12acd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:37.939 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.937+0000 7f81f2926640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f81ec130a60 0x7f81ec12b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:37.939 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.937+0000 7f81f1924640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f81ec134b20 0x7f81ec12b210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:37.939 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.937+0000 7f81f2926640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f81ec130a60 0x7f81ec12b750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:52340/0 (socket says 172.21.15.105:52340) 2024-09-21T01:46:37.939 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.937+0000 7f81f2125640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f81ec130370 0x7f81ec12acd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:48654/0 (socket says 172.21.15.105:48654) 2024-09-21T01:46:37.939 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.937+0000 7f81f2926640 1 -- 172.21.15.105:0/930824847 learned_addr learned my addr 172.21.15.105:0/930824847 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:37.939 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.937+0000 7f81f2125640 1 -- 172.21.15.105:0/930824847 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f81ec134b20 msgr2=0x7f81ec12b210 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:37.940 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.937+0000 7f81f2125640 1 --2- 172.21.15.105:0/930824847 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f81ec134b20 0x7f81ec12b210 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:37.940 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.937+0000 7f81f2125640 1 -- 172.21.15.105:0/930824847 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f81ec130a60 msgr2=0x7f81ec12b750 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:37.940 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.937+0000 7f81f2125640 1 --2- 172.21.15.105:0/930824847 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f81ec130a60 0x7f81ec12b750 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:37.940 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.937+0000 7f81f2125640 1 -- 172.21.15.105:0/930824847 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f81ec12cb10 con 0x7f81ec130370 2024-09-21T01:46:37.940 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.937+0000 7f81f1924640 1 --2- 172.21.15.105:0/930824847 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f81ec134b20 0x7f81ec12b210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:37.940 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.937+0000 7f81f2926640 1 --2- 172.21.15.105:0/930824847 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f81ec130a60 0x7f81ec12b750 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:37.940 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.937+0000 7f81f2125640 1 --2- 172.21.15.105:0/930824847 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f81ec130370 0x7f81ec12acd0 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f81e800c3c0 tx=0x7f81e800c880 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:37.940 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.937+0000 7f81e2ffd640 1 -- 172.21.15.105:0/930824847 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f81e8017070 con 0x7f81ec130370 2024-09-21T01:46:37.940 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.937+0000 7f81e2ffd640 1 -- 172.21.15.105:0/930824847 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f81e80035f0 con 0x7f81ec130370 2024-09-21T01:46:37.940 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.937+0000 7f81f4389640 1 -- 172.21.15.105:0/930824847 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f81ec134710 con 0x7f81ec130370 2024-09-21T01:46:37.940 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.937+0000 7f81e2ffd640 1 -- 172.21.15.105:0/930824847 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f81e8003a40 con 0x7f81ec130370 2024-09-21T01:46:37.941 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.937+0000 7f81f4389640 1 -- 172.21.15.105:0/930824847 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f81ec12c580 con 0x7f81ec130370 2024-09-21T01:46:37.942 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.938+0000 7f81f4389640 1 -- 172.21.15.105:0/930824847 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f81ec12bd50 con 0x7f81ec130370 2024-09-21T01:46:37.942 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.939+0000 7f81e2ffd640 1 -- 172.21.15.105:0/930824847 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f81e801a070 con 0x7f81ec130370 2024-09-21T01:46:37.942 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.939+0000 7f81e2ffd640 1 -- 172.21.15.105:0/930824847 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f81e8068200 con 0x7f81ec130370 2024-09-21T01:46:37.942 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.939+0000 7f81e2ffd640 1 -- 172.21.15.105:0/930824847 --> v1:172.21.15.105:6800/249246859 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f81e801be50 con 0x7f81d0057df0 2024-09-21T01:46:37.942 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.939+0000 7f81e2ffd640 1 -- 172.21.15.105:0/930824847 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7f81e806f050 con 0x7f81ec130370 2024-09-21T01:46:37.942 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.940+0000 7f81e2ffd640 1 -- 172.21.15.105:0/930824847 <== osd.0 v1:172.21.15.105:6800/249246859 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f81e801be50 con 0x7f81d0057df0 2024-09-21T01:46:37.952 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.952+0000 7f81f4389640 1 -- 172.21.15.105:0/930824847 --> v1:172.21.15.105:6800/249246859 -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) -- 0x7f81ec063a50 con 0x7f81d0057df0 2024-09-21T01:46:37.953 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.953+0000 7f81e2ffd640 1 -- 172.21.15.105:0/930824847 <== osd.0 v1:172.21.15.105:6800/249246859 2 ==== command_reply(tid 2: 0 ) ==== 8+0+565 (unknown 0 0 125856537) 0x7f81ec063a50 con 0x7f81d0057df0 2024-09-21T01:46:37.953 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-21T01:46:37.953 INFO:teuthology.orchestra.run.smithi105.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2024-09-21T01:46:37.953 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-21T01:46:37.955 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.953+0000 7f81f4389640 1 -- 172.21.15.105:0/930824847 >> v1:172.21.15.105:6800/249246859 conn(0x7f81d0057df0 legacy=0x7f81d005a280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:37.955 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.953+0000 7f81f4389640 1 -- 172.21.15.105:0/930824847 >> v1:172.21.15.105:6816/2838005468 conn(0x7f81d00514b0 legacy=0x7f81d00539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:37.955 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.953+0000 7f81f4389640 1 -- 172.21.15.105:0/930824847 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f81ec130370 msgr2=0x7f81ec12acd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:37.955 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.953+0000 7f81f4389640 1 --2- 172.21.15.105:0/930824847 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f81ec130370 0x7f81ec12acd0 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f81e800c3c0 tx=0x7f81e800c880 comp rx=0 tx=0).stop 2024-09-21T01:46:37.955 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.953+0000 7f81f2926640 1 -- 172.21.15.105:0/930824847 reap_dead start 2024-09-21T01:46:37.955 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.953+0000 7f81f4389640 1 -- 172.21.15.105:0/930824847 shutdown_connections 2024-09-21T01:46:37.955 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.953+0000 7f81f4389640 1 -- 172.21.15.105:0/930824847 >> 172.21.15.105:0/930824847 conn(0x7f81ec0a2260 msgr2=0x7f81ec1362a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:37.955 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.953+0000 7f81f4389640 1 -- 172.21.15.105:0/930824847 shutdown_connections 2024-09-21T01:46:37.955 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:37.953+0000 7f81f4389640 1 -- 172.21.15.105:0/930824847 wait complete. 2024-09-21T01:46:37.964 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 0 2024-09-21T01:46:38.075 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.075+0000 7f046a22d640 1 -- 172.21.15.105:0/3455306168 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0464073660 msgr2=0x7f0464099250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.076 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.075+0000 7f046a22d640 1 --2- 172.21.15.105:0/3455306168 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0464073660 0x7f0464099250 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f0458009830 tx=0x7f045802a3f0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.076 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.075+0000 7f046a22d640 1 -- 172.21.15.105:0/3455306168 shutdown_connections 2024-09-21T01:46:38.076 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.075+0000 7f046a22d640 1 --2- 172.21.15.105:0/3455306168 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0464073660 0x7f0464099250 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.076 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.075+0000 7f046a22d640 1 --2- 172.21.15.105:0/3455306168 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0464072d10 0x7f0464073120 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.076 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.075+0000 7f046a22d640 1 -- 172.21.15.105:0/3455306168 >> 172.21.15.105:0/3455306168 conn(0x7f04640a01d0 msgr2=0x7f046409c6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:38.076 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.075+0000 7f046a22d640 1 -- 172.21.15.105:0/3455306168 shutdown_connections 2024-09-21T01:46:38.076 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.075+0000 7f046a22d640 1 -- 172.21.15.105:0/3455306168 wait complete. 2024-09-21T01:46:38.078 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.076+0000 7f046a22d640 1 Processor -- start 2024-09-21T01:46:38.078 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.076+0000 7f046a22d640 1 -- start start 2024-09-21T01:46:38.078 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.077+0000 7f046a22d640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0464072d10 0x7f0464127f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:38.078 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.077+0000 7f046a22d640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0464073660 0x7f04641284c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:38.079 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.077+0000 7f046a22d640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0464099790 0x7f0464128a00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:38.079 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.077+0000 7f046a22d640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f04641368f0 con 0x7f0464072d10 2024-09-21T01:46:38.079 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.077+0000 7f046a22d640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f0464136770 con 0x7f0464099790 2024-09-21T01:46:38.079 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.077+0000 7f046a22d640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f0464136a70 con 0x7f0464073660 2024-09-21T01:46:38.079 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.077+0000 7f0462ffd640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0464073660 0x7f04641284c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:38.079 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.077+0000 7f04637fe640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0464072d10 0x7f0464127f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:38.079 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.077+0000 7f0463fff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0464099790 0x7f0464128a00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:38.079 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.077+0000 7f0462ffd640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0464073660 0x7f04641284c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:48660/0 (socket says 172.21.15.105:48660) 2024-09-21T01:46:38.079 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.077+0000 7f04637fe640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0464072d10 0x7f0464127f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:52356/0 (socket says 172.21.15.105:52356) 2024-09-21T01:46:38.079 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.077+0000 7f0462ffd640 1 -- 172.21.15.105:0/2334979686 learned_addr learned my addr 172.21.15.105:0/2334979686 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:38.079 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.077+0000 7f0462ffd640 1 -- 172.21.15.105:0/2334979686 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0464099790 msgr2=0x7f0464128a00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.079 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.077+0000 7f0462ffd640 1 --2- 172.21.15.105:0/2334979686 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0464099790 0x7f0464128a00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.079 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.077+0000 7f0462ffd640 1 -- 172.21.15.105:0/2334979686 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0464072d10 msgr2=0x7f0464127f80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.080 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.077+0000 7f0462ffd640 1 --2- 172.21.15.105:0/2334979686 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0464072d10 0x7f0464127f80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.080 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.077+0000 7f0462ffd640 1 -- 172.21.15.105:0/2334979686 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f046412af50 con 0x7f0464073660 2024-09-21T01:46:38.080 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.077+0000 7f0462ffd640 1 --2- 172.21.15.105:0/2334979686 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0464073660 0x7f04641284c0 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f0458009950 tx=0x7f0458032c10 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:38.080 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.077+0000 7f044bfff640 1 -- 172.21.15.105:0/2334979686 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0458040070 con 0x7f0464073660 2024-09-21T01:46:38.080 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.077+0000 7f044bfff640 1 -- 172.21.15.105:0/2334979686 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f0458005310 con 0x7f0464073660 2024-09-21T01:46:38.080 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.078+0000 7f046a22d640 1 -- 172.21.15.105:0/2334979686 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f046412a900 con 0x7f0464073660 2024-09-21T01:46:38.080 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.078+0000 7f044bfff640 1 -- 172.21.15.105:0/2334979686 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f0458005c90 con 0x7f0464073660 2024-09-21T01:46:38.080 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.078+0000 7f046a22d640 1 -- 172.21.15.105:0/2334979686 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f04641cbe90 con 0x7f0464073660 2024-09-21T01:46:38.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.079+0000 7f046a22d640 1 -- 172.21.15.105:0/2334979686 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f0464129850 con 0x7f0464073660 2024-09-21T01:46:38.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.079+0000 7f044bfff640 1 -- 172.21.15.105:0/2334979686 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f0458039040 con 0x7f0464073660 2024-09-21T01:46:38.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.079+0000 7f044bfff640 1 -- 172.21.15.105:0/2334979686 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f045808a180 con 0x7f0464073660 2024-09-21T01:46:38.083 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.083+0000 7f044bfff640 1 -- 172.21.15.105:0/2334979686 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f0464129850 con 0x7f0464073660 2024-09-21T01:46:38.231 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.231+0000 7f046a22d640 1 -- 172.21.15.105:0/2334979686 --> v1:172.21.15.105:6816/2838005468 -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "0", "target": ["mon-mgr", ""]}) -- 0x7f0464073120 con 0x7f04500514b0 2024-09-21T01:46:38.231 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.231+0000 7f044bfff640 1 -- 172.21.15.105:0/2334979686 <== mgr.14125 v1:172.21.15.105:6816/2838005468 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 0 to deep-scrub) ==== 41+0+0 (unknown 2057319476 0 0) 0x7f0464073120 con 0x7f04500514b0 2024-09-21T01:46:38.232 INFO:teuthology.orchestra.run.smithi105.stderr:instructed osd(s) 0 to deep-scrub 2024-09-21T01:46:38.234 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.233+0000 7f046a22d640 1 -- 172.21.15.105:0/2334979686 >> v1:172.21.15.105:6816/2838005468 conn(0x7f04500514b0 legacy=0x7f04500539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.234 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.233+0000 7f046a22d640 1 -- 172.21.15.105:0/2334979686 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0464073660 msgr2=0x7f04641284c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.234 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.233+0000 7f046a22d640 1 --2- 172.21.15.105:0/2334979686 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0464073660 0x7f04641284c0 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f0458009950 tx=0x7f0458032c10 comp rx=0 tx=0).stop 2024-09-21T01:46:38.234 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.233+0000 7f046a22d640 1 -- 172.21.15.105:0/2334979686 shutdown_connections 2024-09-21T01:46:38.234 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.233+0000 7f046a22d640 1 --2- 172.21.15.105:0/2334979686 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f0464099790 0x7f0464128a00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.234 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.233+0000 7f046a22d640 1 --2- 172.21.15.105:0/2334979686 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f0464073660 0x7f04641284c0 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.234 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.233+0000 7f046a22d640 1 --2- 172.21.15.105:0/2334979686 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f0464072d10 0x7f0464127f80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.234 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.233+0000 7f046a22d640 1 -- 172.21.15.105:0/2334979686 >> 172.21.15.105:0/2334979686 conn(0x7f04640a01d0 msgr2=0x7f04640715b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:38.234 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.233+0000 7f046a22d640 1 -- 172.21.15.105:0/2334979686 shutdown_connections 2024-09-21T01:46:38.234 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.233+0000 7f046a22d640 1 -- 172.21.15.105:0/2334979686 wait complete. 2024-09-21T01:46:38.245 INFO:tasks.ceph:Scrubbing osd.1 2024-09-21T01:46:38.245 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.1 config set osd_debug_deep_scrub_sleep 0 2024-09-21T01:46:38.359 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.358+0000 7f6d025ae640 1 -- 172.21.15.105:0/1699570650 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6cfc12a5f0 msgr2=0x7f6cfc12aa00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.359 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.358+0000 7f6d025ae640 1 --2- 172.21.15.105:0/1699570650 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6cfc12a5f0 0x7f6cfc12aa00 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f6cf0009830 tx=0x7f6cf002a3e0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.359 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.358+0000 7f6d025ae640 1 -- 172.21.15.105:0/1699570650 shutdown_connections 2024-09-21T01:46:38.359 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.358+0000 7f6d025ae640 1 --2- 172.21.15.105:0/1699570650 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6cfc12d4b0 0x7f6cfc12d8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.359 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.358+0000 7f6d025ae640 1 --2- 172.21.15.105:0/1699570650 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6cfc12a5f0 0x7f6cfc12aa00 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.359 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.358+0000 7f6d025ae640 1 --2- 172.21.15.105:0/1699570650 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6cfc09ceb0 0x7f6cfc12a0b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.359 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.358+0000 7f6d025ae640 1 -- 172.21.15.105:0/1699570650 >> 172.21.15.105:0/1699570650 conn(0x7f6cfc0a00f0 msgr2=0x7f6cfc09c5c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:38.359 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.358+0000 7f6d025ae640 1 -- 172.21.15.105:0/1699570650 shutdown_connections 2024-09-21T01:46:38.359 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.358+0000 7f6d025ae640 1 -- 172.21.15.105:0/1699570650 wait complete. 2024-09-21T01:46:38.361 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.359+0000 7f6d025ae640 1 Processor -- start 2024-09-21T01:46:38.361 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.359+0000 7f6d025ae640 1 -- start start 2024-09-21T01:46:38.361 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.360+0000 7f6d025ae640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6cfc09ceb0 0x7f6cfc098740 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:38.361 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.360+0000 7f6d025ae640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6cfc12a5f0 0x7f6cfc098c80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:38.361 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.360+0000 7f6d025ae640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6cfc12d4b0 0x7f6cfc0991c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:38.362 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.360+0000 7f6d025ae640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f6cfc135310 con 0x7f6cfc12a5f0 2024-09-21T01:46:38.362 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.360+0000 7f6d025ae640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f6cfc135190 con 0x7f6cfc12d4b0 2024-09-21T01:46:38.362 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.360+0000 7f6d025ae640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f6cfc135490 con 0x7f6cfc09ceb0 2024-09-21T01:46:38.362 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.360+0000 7f6cfbfff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6cfc09ceb0 0x7f6cfc098740 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:38.362 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.360+0000 7f6cfb7fe640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6cfc12a5f0 0x7f6cfc098c80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:38.362 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.360+0000 7f6cfbfff640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6cfc09ceb0 0x7f6cfc098740 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:48678/0 (socket says 172.21.15.105:48678) 2024-09-21T01:46:38.362 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.360+0000 7f6d00b4b640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6cfc12d4b0 0x7f6cfc0991c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:38.362 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.360+0000 7f6cfbfff640 1 -- 172.21.15.105:0/590157520 learned_addr learned my addr 172.21.15.105:0/590157520 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:38.362 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.360+0000 7f6d00b4b640 1 -- 172.21.15.105:0/590157520 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6cfc09ceb0 msgr2=0x7f6cfc098740 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.362 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.360+0000 7f6d00b4b640 1 --2- 172.21.15.105:0/590157520 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6cfc09ceb0 0x7f6cfc098740 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.362 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.360+0000 7f6d00b4b640 1 -- 172.21.15.105:0/590157520 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6cfc12a5f0 msgr2=0x7f6cfc098c80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.362 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.360+0000 7f6d00b4b640 1 --2- 172.21.15.105:0/590157520 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f6cfc12a5f0 0x7f6cfc098c80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.362 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.360+0000 7f6d00b4b640 1 -- 172.21.15.105:0/590157520 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f6cfc08c0f0 con 0x7f6cfc12d4b0 2024-09-21T01:46:38.363 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.360+0000 7f6cfbfff640 1 --2- 172.21.15.105:0/590157520 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f6cfc09ceb0 0x7f6cfc098740 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:38.363 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.360+0000 7f6d00b4b640 1 --2- 172.21.15.105:0/590157520 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6cfc12d4b0 0x7f6cfc0991c0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f6cfc0741e0 tx=0x7f6cf400c2d0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:38.363 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.360+0000 7f6cf8ff9640 1 -- 172.21.15.105:0/590157520 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6cf4003970 con 0x7f6cfc12d4b0 2024-09-21T01:46:38.363 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.361+0000 7f6cf8ff9640 1 -- 172.21.15.105:0/590157520 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f6cf4003b10 con 0x7f6cfc12d4b0 2024-09-21T01:46:38.363 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.361+0000 7f6cf8ff9640 1 -- 172.21.15.105:0/590157520 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f6cf4004020 con 0x7f6cfc12d4b0 2024-09-21T01:46:38.363 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.361+0000 7f6d025ae640 1 -- 172.21.15.105:0/590157520 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f6cfc133760 con 0x7f6cfc12d4b0 2024-09-21T01:46:38.363 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.361+0000 7f6d025ae640 1 -- 172.21.15.105:0/590157520 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f6cfc1316a0 con 0x7f6cfc12d4b0 2024-09-21T01:46:38.364 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.361+0000 7f6d025ae640 1 -- 172.21.15.105:0/590157520 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f6cfc12aa00 con 0x7f6cfc12d4b0 2024-09-21T01:46:38.364 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.362+0000 7f6cf8ff9640 1 -- 172.21.15.105:0/590157520 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f6cf40041c0 con 0x7f6cfc12d4b0 2024-09-21T01:46:38.364 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.362+0000 7f6cf8ff9640 1 -- 172.21.15.105:0/590157520 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f6cf4069ae0 con 0x7f6cfc12d4b0 2024-09-21T01:46:38.364 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.362+0000 7f6cf8ff9640 1 -- 172.21.15.105:0/590157520 --> v1:172.21.15.105:6804/1896987326 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f6cf4005a20 con 0x7f6ce8057df0 2024-09-21T01:46:38.364 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.362+0000 7f6cf8ff9640 1 -- 172.21.15.105:0/590157520 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7f6cf406e2c0 con 0x7f6cfc12d4b0 2024-09-21T01:46:38.364 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.363+0000 7f6cf8ff9640 1 -- 172.21.15.105:0/590157520 <== osd.1 v1:172.21.15.105:6804/1896987326 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f6cf4005a20 con 0x7f6ce8057df0 2024-09-21T01:46:38.376 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.375+0000 7f6d025ae640 1 -- 172.21.15.105:0/590157520 --> v1:172.21.15.105:6804/1896987326 -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) -- 0x7f6cfc131bb0 con 0x7f6ce8057df0 2024-09-21T01:46:38.376 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.376+0000 7f6cf8ff9640 1 -- 172.21.15.105:0/590157520 <== osd.1 v1:172.21.15.105:6804/1896987326 2 ==== command_reply(tid 2: 0 ) ==== 8+0+565 (unknown 0 0 125856537) 0x7f6cfc131bb0 con 0x7f6ce8057df0 2024-09-21T01:46:38.376 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-21T01:46:38.376 INFO:teuthology.orchestra.run.smithi105.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2024-09-21T01:46:38.376 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-21T01:46:38.377 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.376+0000 7f6d025ae640 1 -- 172.21.15.105:0/590157520 >> v1:172.21.15.105:6804/1896987326 conn(0x7f6ce8057df0 legacy=0x7f6ce805a280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.377 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.376+0000 7f6d025ae640 1 -- 172.21.15.105:0/590157520 >> v1:172.21.15.105:6816/2838005468 conn(0x7f6ce80514b0 legacy=0x7f6ce80539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.378 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.376+0000 7f6d025ae640 1 -- 172.21.15.105:0/590157520 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6cfc12d4b0 msgr2=0x7f6cfc0991c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.378 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.376+0000 7f6d025ae640 1 --2- 172.21.15.105:0/590157520 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f6cfc12d4b0 0x7f6cfc0991c0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7f6cfc0741e0 tx=0x7f6cf400c2d0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.378 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.376+0000 7f6d00b4b640 1 -- 172.21.15.105:0/590157520 reap_dead start 2024-09-21T01:46:38.378 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.376+0000 7f6d025ae640 1 -- 172.21.15.105:0/590157520 shutdown_connections 2024-09-21T01:46:38.378 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.376+0000 7f6d025ae640 1 -- 172.21.15.105:0/590157520 >> 172.21.15.105:0/590157520 conn(0x7f6cfc0a00f0 msgr2=0x7f6cfc072950 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:38.378 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.376+0000 7f6d025ae640 1 -- 172.21.15.105:0/590157520 shutdown_connections 2024-09-21T01:46:38.378 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.376+0000 7f6d025ae640 1 -- 172.21.15.105:0/590157520 wait complete. 2024-09-21T01:46:38.388 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 1 2024-09-21T01:46:38.499 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.498+0000 7f1048e13640 1 -- 172.21.15.105:0/3739733530 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f104412cd40 msgr2=0x7f10441294e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.499 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.498+0000 7f1048e13640 1 --2- 172.21.15.105:0/3739733530 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f104412cd40 0x7f10441294e0 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7f103c005fd0 tx=0x7f103c02a4b0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.499 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.498+0000 7f1048e13640 1 -- 172.21.15.105:0/3739733530 shutdown_connections 2024-09-21T01:46:38.499 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.498+0000 7f1048e13640 1 --2- 172.21.15.105:0/3739733530 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f104412cd40 0x7f10441294e0 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.499 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.498+0000 7f1048e13640 1 --2- 172.21.15.105:0/3739733530 >> v2:172.21.15.105:3300/0 conn(0x7f10440727d0 0x7f104412c800 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.499 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.498+0000 7f1048e13640 1 -- 172.21.15.105:0/3739733530 >> 172.21.15.105:0/3739733530 conn(0x7f104409eb40 msgr2=0x7f104409ef50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:38.500 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.498+0000 7f1048e13640 1 -- 172.21.15.105:0/3739733530 shutdown_connections 2024-09-21T01:46:38.500 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.498+0000 7f1048e13640 1 -- 172.21.15.105:0/3739733530 wait complete. 2024-09-21T01:46:38.503 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.499+0000 7f1048e13640 1 Processor -- start 2024-09-21T01:46:38.503 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.499+0000 7f1048e13640 1 -- start start 2024-09-21T01:46:38.503 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.500+0000 7f1048e13640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f10440727d0 0x7f10441306d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:38.503 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.500+0000 7f1048e13640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f104412c2d0 0x7f1044130c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:38.503 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.500+0000 7f1048e13640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f104412cd40 0x7f1044131150 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:38.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.500+0000 7f1048e13640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f1044135280 con 0x7f104412c2d0 2024-09-21T01:46:38.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.500+0000 7f1048e13640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f1044135100 con 0x7f10440727d0 2024-09-21T01:46:38.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.500+0000 7f1048e13640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f1044135400 con 0x7f104412cd40 2024-09-21T01:46:38.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.500+0000 7f104259c640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f10440727d0 0x7f10441306d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:38.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.500+0000 7f1041d9b640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f104412c2d0 0x7f1044130c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:38.504 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.500+0000 7f104259c640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f10440727d0 0x7f10441306d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:50888/0 (socket says 172.21.15.105:50888) 2024-09-21T01:46:38.505 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.500+0000 7f104259c640 1 -- 172.21.15.105:0/2238865125 learned_addr learned my addr 172.21.15.105:0/2238865125 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:38.505 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.500+0000 7f1042d9d640 1 --2- 172.21.15.105:0/2238865125 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f104412cd40 0x7f1044131150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:38.505 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.500+0000 7f104259c640 1 -- 172.21.15.105:0/2238865125 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f104412cd40 msgr2=0x7f1044131150 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.505 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.501+0000 7f104259c640 1 --2- 172.21.15.105:0/2238865125 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f104412cd40 0x7f1044131150 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.505 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.501+0000 7f104259c640 1 -- 172.21.15.105:0/2238865125 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f104412c2d0 msgr2=0x7f1044130c10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.505 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.501+0000 7f104259c640 1 --2- 172.21.15.105:0/2238865125 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f104412c2d0 0x7f1044130c10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.506 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.501+0000 7f104259c640 1 -- 172.21.15.105:0/2238865125 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f104406f340 con 0x7f10440727d0 2024-09-21T01:46:38.506 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.501+0000 7f1041d9b640 1 --2- 172.21.15.105:0/2238865125 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f104412c2d0 0x7f1044130c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:38.506 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.501+0000 7f104259c640 1 --2- 172.21.15.105:0/2238865125 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f10440727d0 0x7f10441306d0 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f103400ece0 tx=0x7f103400c6a0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:38.506 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.501+0000 7f1032ffd640 1 -- 172.21.15.105:0/2238865125 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1034003b70 con 0x7f10440727d0 2024-09-21T01:46:38.506 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.501+0000 7f1032ffd640 1 -- 172.21.15.105:0/2238865125 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f1034003d10 con 0x7f10440727d0 2024-09-21T01:46:38.506 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.501+0000 7f1048e13640 1 -- 172.21.15.105:0/2238865125 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f1044098140 con 0x7f10440727d0 2024-09-21T01:46:38.507 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.501+0000 7f1032ffd640 1 -- 172.21.15.105:0/2238865125 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f1034004280 con 0x7f10440727d0 2024-09-21T01:46:38.507 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.501+0000 7f1048e13640 1 -- 172.21.15.105:0/2238865125 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f10441c36e0 con 0x7f10440727d0 2024-09-21T01:46:38.507 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.502+0000 7f1048e13640 1 -- 172.21.15.105:0/2238865125 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f104412c800 con 0x7f10440727d0 2024-09-21T01:46:38.507 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.502+0000 7f1032ffd640 1 -- 172.21.15.105:0/2238865125 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f1034004420 con 0x7f10440727d0 2024-09-21T01:46:38.507 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.503+0000 7f1032ffd640 1 -- 172.21.15.105:0/2238865125 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f1034069c90 con 0x7f10440727d0 2024-09-21T01:46:38.508 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.506+0000 7f1032ffd640 1 -- 172.21.15.105:0/2238865125 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f1034068d40 con 0x7f10440727d0 2024-09-21T01:46:38.657 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.656+0000 7f1048e13640 1 -- 172.21.15.105:0/2238865125 --> v1:172.21.15.105:6816/2838005468 -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "1", "target": ["mon-mgr", ""]}) -- 0x7f104406a780 con 0x7f10240514b0 2024-09-21T01:46:38.657 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.656+0000 7f1032ffd640 1 -- 172.21.15.105:0/2238865125 <== mgr.14125 v1:172.21.15.105:6816/2838005468 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 1 to deep-scrub) ==== 41+0+0 (unknown 489605429 0 0) 0x7f104406a780 con 0x7f10240514b0 2024-09-21T01:46:38.657 INFO:teuthology.orchestra.run.smithi105.stderr:instructed osd(s) 1 to deep-scrub 2024-09-21T01:46:38.659 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.658+0000 7f1048e13640 1 -- 172.21.15.105:0/2238865125 >> v1:172.21.15.105:6816/2838005468 conn(0x7f10240514b0 legacy=0x7f10240539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.660 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.658+0000 7f1048e13640 1 -- 172.21.15.105:0/2238865125 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f10440727d0 msgr2=0x7f10441306d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.660 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.658+0000 7f1048e13640 1 --2- 172.21.15.105:0/2238865125 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f10440727d0 0x7f10441306d0 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f103400ece0 tx=0x7f103400c6a0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.660 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.658+0000 7f1048e13640 1 -- 172.21.15.105:0/2238865125 shutdown_connections 2024-09-21T01:46:38.660 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.658+0000 7f1048e13640 1 --2- 172.21.15.105:0/2238865125 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f104412cd40 0x7f1044131150 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.660 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.658+0000 7f1048e13640 1 --2- 172.21.15.105:0/2238865125 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f104412c2d0 0x7f1044130c10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.660 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.658+0000 7f1048e13640 1 --2- 172.21.15.105:0/2238865125 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f10440727d0 0x7f10441306d0 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.661 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.658+0000 7f1048e13640 1 -- 172.21.15.105:0/2238865125 >> 172.21.15.105:0/2238865125 conn(0x7f104409eb40 msgr2=0x7f10440985f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:38.661 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.658+0000 7f1048e13640 1 -- 172.21.15.105:0/2238865125 shutdown_connections 2024-09-21T01:46:38.661 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.658+0000 7f1048e13640 1 -- 172.21.15.105:0/2238865125 wait complete. 2024-09-21T01:46:38.670 INFO:tasks.ceph:Scrubbing osd.2 2024-09-21T01:46:38.670 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 config set osd_debug_deep_scrub_sleep 0 2024-09-21T01:46:38.785 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.784+0000 7f56d740a640 1 -- 172.21.15.105:0/2325628502 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f56d01353b0 msgr2=0x7f56d01377d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.785 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.784+0000 7f56d740a640 1 --2- 172.21.15.105:0/2325628502 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f56d01353b0 0x7f56d01377d0 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f56c8005fd0 tx=0x7f56c802aa10 comp rx=0 tx=0).stop 2024-09-21T01:46:38.785 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.784+0000 7f56d740a640 1 -- 172.21.15.105:0/2325628502 shutdown_connections 2024-09-21T01:46:38.785 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.784+0000 7f56d740a640 1 --2- 172.21.15.105:0/2325628502 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f56d01353b0 0x7f56d01377d0 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.785 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.784+0000 7f56d740a640 1 --2- 172.21.15.105:0/2325628502 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f56d0132a50 0x7f56d0134e70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.785 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.784+0000 7f56d740a640 1 -- 172.21.15.105:0/2325628502 >> 172.21.15.105:0/2325628502 conn(0x7f56d00a1f40 msgr2=0x7f56d00a2350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:38.786 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.784+0000 7f56d740a640 1 -- 172.21.15.105:0/2325628502 shutdown_connections 2024-09-21T01:46:38.786 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.784+0000 7f56d740a640 1 -- 172.21.15.105:0/2325628502 wait complete. 2024-09-21T01:46:38.787 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.785+0000 7f56d740a640 1 Processor -- start 2024-09-21T01:46:38.788 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.785+0000 7f56d740a640 1 -- start start 2024-09-21T01:46:38.788 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.786+0000 7f56d740a640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f56d012c420 0x7f56d012d4c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:38.788 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.786+0000 7f56d740a640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f56d0132a50 0x7f56d012da00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:38.788 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.786+0000 7f56d740a640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f56d01353b0 0x7f56d012df40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:38.788 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.786+0000 7f56d740a640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f56d0138790 con 0x7f56d0132a50 2024-09-21T01:46:38.788 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.786+0000 7f56d740a640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f56d0138610 con 0x7f56d01353b0 2024-09-21T01:46:38.788 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.786+0000 7f56d740a640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f56d0138910 con 0x7f56d012c420 2024-09-21T01:46:38.788 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.786+0000 7f56d49a5640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f56d0132a50 0x7f56d012da00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:38.788 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.786+0000 7f56d59a7640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f56d01353b0 0x7f56d012df40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:38.788 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.786+0000 7f56d51a6640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f56d012c420 0x7f56d012d4c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:38.788 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.786+0000 7f56d59a7640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f56d01353b0 0x7f56d012df40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:50900/0 (socket says 172.21.15.105:50900) 2024-09-21T01:46:38.788 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.786+0000 7f56d49a5640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f56d0132a50 0x7f56d012da00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:52412/0 (socket says 172.21.15.105:52412) 2024-09-21T01:46:38.789 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.786+0000 7f56d59a7640 1 -- 172.21.15.105:0/3862893886 learned_addr learned my addr 172.21.15.105:0/3862893886 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:38.789 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.786+0000 7f56d59a7640 1 -- 172.21.15.105:0/3862893886 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f56d012c420 msgr2=0x7f56d012d4c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.789 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.786+0000 7f56d59a7640 1 --2- 172.21.15.105:0/3862893886 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f56d012c420 0x7f56d012d4c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.789 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.786+0000 7f56d59a7640 1 -- 172.21.15.105:0/3862893886 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f56d0132a50 msgr2=0x7f56d012da00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.789 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.786+0000 7f56d59a7640 1 --2- 172.21.15.105:0/3862893886 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f56d0132a50 0x7f56d012da00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.789 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.786+0000 7f56d59a7640 1 -- 172.21.15.105:0/3862893886 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f56d012f300 con 0x7f56d01353b0 2024-09-21T01:46:38.789 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.787+0000 7f56d49a5640 1 --2- 172.21.15.105:0/3862893886 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f56d0132a50 0x7f56d012da00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:38.789 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.787+0000 7f56d59a7640 1 --2- 172.21.15.105:0/3862893886 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f56d01353b0 0x7f56d012df40 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f56c8006010 tx=0x7f56c8009360 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:38.789 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.787+0000 7f56c5ffb640 1 -- 172.21.15.105:0/3862893886 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f56c8046070 con 0x7f56d01353b0 2024-09-21T01:46:38.789 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.787+0000 7f56c5ffb640 1 -- 172.21.15.105:0/3862893886 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f56c8004540 con 0x7f56d01353b0 2024-09-21T01:46:38.789 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.787+0000 7f56d740a640 1 -- 172.21.15.105:0/3862893886 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f56d012e8c0 con 0x7f56d01353b0 2024-09-21T01:46:38.789 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.787+0000 7f56c5ffb640 1 -- 172.21.15.105:0/3862893886 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f56c8033070 con 0x7f56d01353b0 2024-09-21T01:46:38.789 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.787+0000 7f56d740a640 1 -- 172.21.15.105:0/3862893886 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f56d009ebe0 con 0x7f56d01353b0 2024-09-21T01:46:38.790 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.788+0000 7f56d740a640 1 -- 172.21.15.105:0/3862893886 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f56d012a3f0 con 0x7f56d01353b0 2024-09-21T01:46:38.790 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.788+0000 7f56c5ffb640 1 -- 172.21.15.105:0/3862893886 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f56c8049070 con 0x7f56d01353b0 2024-09-21T01:46:38.790 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.789+0000 7f56c5ffb640 1 -- 172.21.15.105:0/3862893886 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f56c808a190 con 0x7f56d01353b0 2024-09-21T01:46:38.790 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.789+0000 7f56c5ffb640 1 -- 172.21.15.105:0/3862893886 --> v1:172.21.15.105:6808/2933198036 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f56b405baf0 con 0x7f56b4057df0 2024-09-21T01:46:38.790 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.789+0000 7f56c5ffb640 1 -- 172.21.15.105:0/3862893886 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7f56c8091050 con 0x7f56d01353b0 2024-09-21T01:46:38.791 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.790+0000 7f56c5ffb640 1 -- 172.21.15.105:0/3862893886 <== osd.2 v1:172.21.15.105:6808/2933198036 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f56b405baf0 con 0x7f56b4057df0 2024-09-21T01:46:38.802 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.802+0000 7f56d740a640 1 -- 172.21.15.105:0/3862893886 --> v1:172.21.15.105:6808/2933198036 -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) -- 0x7f56d0133160 con 0x7f56b4057df0 2024-09-21T01:46:38.803 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.802+0000 7f56c5ffb640 1 -- 172.21.15.105:0/3862893886 <== osd.2 v1:172.21.15.105:6808/2933198036 2 ==== command_reply(tid 2: 0 ) ==== 8+0+565 (unknown 0 0 125856537) 0x7f56d0133160 con 0x7f56b4057df0 2024-09-21T01:46:38.803 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-21T01:46:38.803 INFO:teuthology.orchestra.run.smithi105.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2024-09-21T01:46:38.803 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-21T01:46:38.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.803+0000 7f56d740a640 1 -- 172.21.15.105:0/3862893886 >> v1:172.21.15.105:6808/2933198036 conn(0x7f56b4057df0 legacy=0x7f56b405a280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.803+0000 7f56d740a640 1 -- 172.21.15.105:0/3862893886 >> v1:172.21.15.105:6816/2838005468 conn(0x7f56b40514b0 legacy=0x7f56b40539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.803+0000 7f56d740a640 1 -- 172.21.15.105:0/3862893886 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f56d01353b0 msgr2=0x7f56d012df40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.803+0000 7f56d740a640 1 --2- 172.21.15.105:0/3862893886 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f56d01353b0 0x7f56d012df40 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f56c8006010 tx=0x7f56c8009360 comp rx=0 tx=0).stop 2024-09-21T01:46:38.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.803+0000 7f56d59a7640 1 -- 172.21.15.105:0/3862893886 reap_dead start 2024-09-21T01:46:38.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.803+0000 7f56d740a640 1 -- 172.21.15.105:0/3862893886 shutdown_connections 2024-09-21T01:46:38.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.803+0000 7f56d740a640 1 -- 172.21.15.105:0/3862893886 >> 172.21.15.105:0/3862893886 conn(0x7f56d00a1f40 msgr2=0x7f56d01342a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:38.804 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.803+0000 7f56d740a640 1 -- 172.21.15.105:0/3862893886 shutdown_connections 2024-09-21T01:46:38.805 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.803+0000 7f56d740a640 1 -- 172.21.15.105:0/3862893886 wait complete. 2024-09-21T01:46:38.814 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 2 2024-09-21T01:46:38.925 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.924+0000 7f27af820640 1 -- 172.21.15.105:0/3087656953 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27a812cde0 msgr2=0x7f27a8071860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.925 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.924+0000 7f27af820640 1 --2- 172.21.15.105:0/3087656953 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27a812cde0 0x7f27a8071860 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f2798009830 tx=0x7f279802a3f0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.925 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.924+0000 7f27af820640 1 -- 172.21.15.105:0/3087656953 shutdown_connections 2024-09-21T01:46:38.925 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.924+0000 7f27af820640 1 --2- 172.21.15.105:0/3087656953 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27a812cde0 0x7f27a8071860 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.925 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.924+0000 7f27af820640 1 --2- 172.21.15.105:0/3087656953 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f27a809bce0 0x7f27a812c8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.925 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.924+0000 7f27af820640 1 -- 172.21.15.105:0/3087656953 >> 172.21.15.105:0/3087656953 conn(0x7f27a809f000 msgr2=0x7f27a809b4d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:38.925 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.924+0000 7f27af820640 1 -- 172.21.15.105:0/3087656953 shutdown_connections 2024-09-21T01:46:38.925 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.924+0000 7f27af820640 1 -- 172.21.15.105:0/3087656953 wait complete. 2024-09-21T01:46:38.928 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.925+0000 7f27af820640 1 Processor -- start 2024-09-21T01:46:38.928 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.925+0000 7f27af820640 1 -- start start 2024-09-21T01:46:38.929 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.926+0000 7f27af820640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27a8071da0 0x7f27a81c28c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:38.929 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.926+0000 7f27af820640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f27a809bce0 0x7f27a81c2e00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:38.929 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.926+0000 7f27af820640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f27a812cde0 0x7f27a81c3340 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:38.929 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.926+0000 7f27af820640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f27a8135470 con 0x7f27a8071da0 2024-09-21T01:46:38.929 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.926+0000 7f27af820640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f27a81352f0 con 0x7f27a812cde0 2024-09-21T01:46:38.930 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.926+0000 7f27af820640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f27a81355f0 con 0x7f27a809bce0 2024-09-21T01:46:38.930 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.926+0000 7f27acdbb640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f27a809bce0 0x7f27a81c2e00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:38.930 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.926+0000 7f27ad5bc640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27a8071da0 0x7f27a81c28c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:38.930 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.926+0000 7f27addbd640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f27a812cde0 0x7f27a81c3340 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:38.930 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.926+0000 7f27acdbb640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f27a809bce0 0x7f27a81c2e00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:48708/0 (socket says 172.21.15.105:48708) 2024-09-21T01:46:38.930 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.926+0000 7f27ad5bc640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27a8071da0 0x7f27a81c28c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:52430/0 (socket says 172.21.15.105:52430) 2024-09-21T01:46:38.930 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.926+0000 7f27acdbb640 1 -- 172.21.15.105:0/618666262 learned_addr learned my addr 172.21.15.105:0/618666262 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:38.930 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.926+0000 7f27acdbb640 1 -- 172.21.15.105:0/618666262 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f27a812cde0 msgr2=0x7f27a81c3340 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.930 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.926+0000 7f27acdbb640 1 --2- 172.21.15.105:0/618666262 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f27a812cde0 0x7f27a81c3340 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.930 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.926+0000 7f27acdbb640 1 -- 172.21.15.105:0/618666262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27a8071da0 msgr2=0x7f27a81c28c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:38.930 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.926+0000 7f27acdbb640 1 --2- 172.21.15.105:0/618666262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27a8071da0 0x7f27a81c28c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:38.930 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.926+0000 7f27acdbb640 1 -- 172.21.15.105:0/618666262 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f27a8131ce0 con 0x7f27a809bce0 2024-09-21T01:46:38.930 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.926+0000 7f27ad5bc640 1 --2- 172.21.15.105:0/618666262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27a8071da0 0x7f27a81c28c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:38.930 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.927+0000 7f27acdbb640 1 --2- 172.21.15.105:0/618666262 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f27a809bce0 0x7f27a81c2e00 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f2798009950 tx=0x7f2798033c20 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:38.931 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.927+0000 7f279dffb640 1 -- 172.21.15.105:0/618666262 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2798005c90 con 0x7f27a809bce0 2024-09-21T01:46:38.931 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.927+0000 7f279dffb640 1 -- 172.21.15.105:0/618666262 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f2798031e50 con 0x7f27a809bce0 2024-09-21T01:46:38.931 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.927+0000 7f27af820640 1 -- 172.21.15.105:0/618666262 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f27a8132900 con 0x7f27a809bce0 2024-09-21T01:46:38.931 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.927+0000 7f279dffb640 1 -- 172.21.15.105:0/618666262 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f2798005260 con 0x7f27a809bce0 2024-09-21T01:46:38.931 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.927+0000 7f27af820640 1 -- 172.21.15.105:0/618666262 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f27a8132e40 con 0x7f27a809bce0 2024-09-21T01:46:38.931 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.928+0000 7f27af820640 1 -- 172.21.15.105:0/618666262 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f27a812c8a0 con 0x7f27a809bce0 2024-09-21T01:46:38.931 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.928+0000 7f279dffb640 1 -- 172.21.15.105:0/618666262 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f2798005400 con 0x7f27a809bce0 2024-09-21T01:46:38.932 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.929+0000 7f279dffb640 1 -- 172.21.15.105:0/618666262 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f279808a0e0 con 0x7f27a809bce0 2024-09-21T01:46:38.933 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:38.933+0000 7f279dffb640 1 -- 172.21.15.105:0/618666262 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f279803f030 con 0x7f27a809bce0 2024-09-21T01:46:39.083 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.083+0000 7f27af820640 1 -- 172.21.15.105:0/618666262 --> v1:172.21.15.105:6816/2838005468 -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "2", "target": ["mon-mgr", ""]}) -- 0x7f27a8130700 con 0x7f278c0514b0 2024-09-21T01:46:39.084 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.083+0000 7f279dffb640 1 -- 172.21.15.105:0/618666262 <== mgr.14125 v1:172.21.15.105:6816/2838005468 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 2 to deep-scrub) ==== 41+0+0 (unknown 3049115190 0 0) 0x7f27a8130700 con 0x7f278c0514b0 2024-09-21T01:46:39.084 INFO:teuthology.orchestra.run.smithi105.stderr:instructed osd(s) 2 to deep-scrub 2024-09-21T01:46:39.085 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.085+0000 7f27af820640 1 -- 172.21.15.105:0/618666262 >> v1:172.21.15.105:6816/2838005468 conn(0x7f278c0514b0 legacy=0x7f278c0539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:39.086 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.085+0000 7f27af820640 1 -- 172.21.15.105:0/618666262 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f27a809bce0 msgr2=0x7f27a81c2e00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:39.086 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.085+0000 7f27af820640 1 --2- 172.21.15.105:0/618666262 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f27a809bce0 0x7f27a81c2e00 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f2798009950 tx=0x7f2798033c20 comp rx=0 tx=0).stop 2024-09-21T01:46:39.086 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.085+0000 7f27af820640 1 -- 172.21.15.105:0/618666262 shutdown_connections 2024-09-21T01:46:39.086 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.085+0000 7f27af820640 1 --2- 172.21.15.105:0/618666262 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f27a812cde0 0x7f27a81c3340 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.086 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.085+0000 7f27af820640 1 --2- 172.21.15.105:0/618666262 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f27a809bce0 0x7f27a81c2e00 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.086 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.085+0000 7f27af820640 1 --2- 172.21.15.105:0/618666262 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f27a8071da0 0x7f27a81c28c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.086 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.085+0000 7f27af820640 1 -- 172.21.15.105:0/618666262 >> 172.21.15.105:0/618666262 conn(0x7f27a809f000 msgr2=0x7f27a8134790 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:39.086 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.085+0000 7f27af820640 1 -- 172.21.15.105:0/618666262 shutdown_connections 2024-09-21T01:46:39.086 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.085+0000 7f27af820640 1 -- 172.21.15.105:0/618666262 wait complete. 2024-09-21T01:46:39.096 INFO:tasks.ceph:Scrubbing osd.3 2024-09-21T01:46:39.097 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.3 config set osd_debug_deep_scrub_sleep 0 2024-09-21T01:46:39.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.210+0000 7f861fc8b640 1 -- 172.21.15.105:0/1193714943 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f861812fc20 msgr2=0x7f8618130030 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:39.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.210+0000 7f861fc8b640 1 --2- 172.21.15.105:0/1193714943 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f861812fc20 0x7f8618130030 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f8610005fd0 tx=0x7f861002a770 comp rx=0 tx=0).stop 2024-09-21T01:46:39.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.210+0000 7f861fc8b640 1 -- 172.21.15.105:0/1193714943 shutdown_connections 2024-09-21T01:46:39.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.210+0000 7f861fc8b640 1 --2- 172.21.15.105:0/1193714943 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f861812fc20 0x7f8618130030 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.210+0000 7f861fc8b640 1 --2- 172.21.15.105:0/1193714943 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f861812cd70 0x7f861812f6e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.210+0000 7f861fc8b640 1 --2- 172.21.15.105:0/1193714943 >> v2:172.21.15.105:3300/0 conn(0x7f861812c420 0x7f861812c830 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.210+0000 7f861fc8b640 1 -- 172.21.15.105:0/1193714943 >> 172.21.15.105:0/1193714943 conn(0x7f86180a1f40 msgr2=0x7f86180a2350 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:39.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.210+0000 7f861fc8b640 1 -- 172.21.15.105:0/1193714943 shutdown_connections 2024-09-21T01:46:39.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.210+0000 7f861fc8b640 1 -- 172.21.15.105:0/1193714943 wait complete. 2024-09-21T01:46:39.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.211+0000 7f861fc8b640 1 Processor -- start 2024-09-21T01:46:39.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.211+0000 7f861fc8b640 1 -- start start 2024-09-21T01:46:39.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.212+0000 7f861fc8b640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f861812c420 0x7f86181c5990 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:39.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.212+0000 7f861fc8b640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f861812cd70 0x7f86181c5ed0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:39.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.212+0000 7f861fc8b640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f861812fc20 0x7f86181c6410 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:39.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.212+0000 7f861fc8b640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f8618137530 con 0x7f861812fc20 2024-09-21T01:46:39.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.212+0000 7f861fc8b640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f86181373b0 con 0x7f861812cd70 2024-09-21T01:46:39.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.212+0000 7f861fc8b640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f86181376b0 con 0x7f861812c420 2024-09-21T01:46:39.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.212+0000 7f861da27640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f861812c420 0x7f86181c5990 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:39.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.212+0000 7f861d226640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f861812cd70 0x7f86181c5ed0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:39.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.212+0000 7f861da27640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f861812c420 0x7f86181c5990 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:48740/0 (socket says 172.21.15.105:48740) 2024-09-21T01:46:39.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.212+0000 7f861da27640 1 -- 172.21.15.105:0/1247492138 learned_addr learned my addr 172.21.15.105:0/1247492138 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:39.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.212+0000 7f861e228640 1 --2- 172.21.15.105:0/1247492138 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f861812fc20 0x7f86181c6410 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:39.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.212+0000 7f861d226640 1 -- 172.21.15.105:0/1247492138 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f861812c420 msgr2=0x7f86181c5990 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:39.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.212+0000 7f861d226640 1 --2- 172.21.15.105:0/1247492138 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f861812c420 0x7f86181c5990 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.212+0000 7f861d226640 1 -- 172.21.15.105:0/1247492138 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f861812fc20 msgr2=0x7f86181c6410 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:39.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.212+0000 7f861d226640 1 --2- 172.21.15.105:0/1247492138 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f861812fc20 0x7f86181c6410 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.212+0000 7f861d226640 1 -- 172.21.15.105:0/1247492138 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8618133ff0 con 0x7f861812cd70 2024-09-21T01:46:39.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.212+0000 7f861da27640 1 --2- 172.21.15.105:0/1247492138 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f861812c420 0x7f86181c5990 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:39.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.212+0000 7f861e228640 1 --2- 172.21.15.105:0/1247492138 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f861812fc20 0x7f86181c6410 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:39.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.212+0000 7f861d226640 1 --2- 172.21.15.105:0/1247492138 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f861812cd70 0x7f86181c5ed0 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f8618074960 tx=0x7f860800ee50 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:39.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.213+0000 7f860e7fc640 1 -- 172.21.15.105:0/1247492138 <== mon.1 v2:172.21.15.105:3301/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8608004440 con 0x7f861812cd70 2024-09-21T01:46:39.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.213+0000 7f860e7fc640 1 -- 172.21.15.105:0/1247492138 <== mon.1 v2:172.21.15.105:3301/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f860800c8c0 con 0x7f861812cd70 2024-09-21T01:46:39.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.213+0000 7f861fc8b640 1 -- 172.21.15.105:0/1247492138 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8618134b70 con 0x7f861812cd70 2024-09-21T01:46:39.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.213+0000 7f860e7fc640 1 -- 172.21.15.105:0/1247492138 <== mon.1 v2:172.21.15.105:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f860800ccf0 con 0x7f861812cd70 2024-09-21T01:46:39.215 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.213+0000 7f861fc8b640 1 -- 172.21.15.105:0/1247492138 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_subscribe({osdmap=0}) -- 0x7f8618135110 con 0x7f861812cd70 2024-09-21T01:46:39.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.214+0000 7f861fc8b640 1 -- 172.21.15.105:0/1247492138 --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_get_version(what=osdmap handle=1) -- 0x7f861812f6e0 con 0x7f861812cd70 2024-09-21T01:46:39.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.214+0000 7f860e7fc640 1 -- 172.21.15.105:0/1247492138 <== mon.1 v2:172.21.15.105:3301/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f86080035f0 con 0x7f861812cd70 2024-09-21T01:46:39.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.214+0000 7f860e7fc640 1 -- 172.21.15.105:0/1247492138 <== mon.1 v2:172.21.15.105:3301/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f8608069b50 con 0x7f861812cd70 2024-09-21T01:46:39.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.214+0000 7f860e7fc640 1 -- 172.21.15.105:0/1247492138 --> v1:172.21.15.105:6812/568702146 -- command(tid 1: {"prefix": "get_command_descriptions"}) -- 0x7f85fc05baf0 con 0x7f85fc057df0 2024-09-21T01:46:39.216 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.214+0000 7f860e7fc640 1 -- 172.21.15.105:0/1247492138 <== mon.1 v2:172.21.15.105:3301/0 6 ==== mon_get_version_reply(handle=1 version=33) ==== 24+0+0 (secure 0 0 0) 0x7f860806e2c0 con 0x7f861812cd70 2024-09-21T01:46:39.217 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.215+0000 7f860e7fc640 1 -- 172.21.15.105:0/1247492138 <== osd.3 v1:172.21.15.105:6812/568702146 1 ==== command_reply(tid 1: 0 ) ==== 8+0+27193 (unknown 0 0 1302949928) 0x7f85fc05baf0 con 0x7f85fc057df0 2024-09-21T01:46:39.228 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.227+0000 7f861fc8b640 1 -- 172.21.15.105:0/1247492138 --> v1:172.21.15.105:6812/568702146 -- command(tid 2: {"prefix": "config set", "var": "osd_debug_deep_scrub_sleep", "val": ["0"]}) -- 0x7f8618129080 con 0x7f85fc057df0 2024-09-21T01:46:39.228 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.228+0000 7f860e7fc640 1 -- 172.21.15.105:0/1247492138 <== osd.3 v1:172.21.15.105:6812/568702146 2 ==== command_reply(tid 2: 0 ) ==== 8+0+565 (unknown 0 0 125856537) 0x7f8618129080 con 0x7f85fc057df0 2024-09-21T01:46:39.228 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-21T01:46:39.228 INFO:teuthology.orchestra.run.smithi105.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2024-09-21T01:46:39.229 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-21T01:46:39.229 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.228+0000 7f861fc8b640 1 -- 172.21.15.105:0/1247492138 >> v1:172.21.15.105:6812/568702146 conn(0x7f85fc057df0 legacy=0x7f85fc05a280 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:39.229 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.228+0000 7f861fc8b640 1 -- 172.21.15.105:0/1247492138 >> v1:172.21.15.105:6816/2838005468 conn(0x7f85fc0514b0 legacy=0x7f85fc0539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:39.229 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.228+0000 7f861fc8b640 1 -- 172.21.15.105:0/1247492138 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f861812cd70 msgr2=0x7f86181c5ed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:39.230 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.228+0000 7f861fc8b640 1 --2- 172.21.15.105:0/1247492138 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f861812cd70 0x7f86181c5ed0 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f8618074960 tx=0x7f860800ee50 comp rx=0 tx=0).stop 2024-09-21T01:46:39.230 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.228+0000 7f861e228640 1 -- 172.21.15.105:0/1247492138 reap_dead start 2024-09-21T01:46:39.230 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.228+0000 7f861fc8b640 1 -- 172.21.15.105:0/1247492138 shutdown_connections 2024-09-21T01:46:39.230 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.228+0000 7f861fc8b640 1 -- 172.21.15.105:0/1247492138 >> 172.21.15.105:0/1247492138 conn(0x7f86180a1f40 msgr2=0x7f861812eb20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:39.230 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.228+0000 7f861fc8b640 1 -- 172.21.15.105:0/1247492138 shutdown_connections 2024-09-21T01:46:39.230 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.228+0000 7f861fc8b640 1 -- 172.21.15.105:0/1247492138 wait complete. 2024-09-21T01:46:39.239 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd deep-scrub 3 2024-09-21T01:46:39.351 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.350+0000 7ff82c334640 1 -- 172.21.15.105:0/2108172497 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff82412e0a0 msgr2=0x7ff824135060 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:39.351 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.350+0000 7ff82c334640 1 --2- 172.21.15.105:0/2108172497 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff82412e0a0 0x7ff824135060 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7ff81c005fd0 tx=0x7ff81c02a780 comp rx=0 tx=0).stop 2024-09-21T01:46:39.351 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.350+0000 7ff82c334640 1 -- 172.21.15.105:0/2108172497 shutdown_connections 2024-09-21T01:46:39.351 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.350+0000 7ff82c334640 1 --2- 172.21.15.105:0/2108172497 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff82412e0a0 0x7ff824135060 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.351 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.350+0000 7ff82c334640 1 --2- 172.21.15.105:0/2108172497 >> v2:172.21.15.105:3300/0 conn(0x7ff82412dc90 0x7ff824132b10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.351 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.350+0000 7ff82c334640 1 -- 172.21.15.105:0/2108172497 >> 172.21.15.105:0/2108172497 conn(0x7ff82409f000 msgr2=0x7ff82409b4d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:39.351 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.350+0000 7ff82c334640 1 -- 172.21.15.105:0/2108172497 shutdown_connections 2024-09-21T01:46:39.352 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.350+0000 7ff82c334640 1 -- 172.21.15.105:0/2108172497 wait complete. 2024-09-21T01:46:39.354 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.351+0000 7ff82c334640 1 Processor -- start 2024-09-21T01:46:39.354 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.351+0000 7ff82c334640 1 -- start start 2024-09-21T01:46:39.354 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.352+0000 7ff82c334640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff824073a70 0x7ff8241346a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:39.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.352+0000 7ff82c334640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff82412dc90 0x7ff824134be0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:39.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.352+0000 7ff82c334640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff82412e0a0 0x7ff8241c3da0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:39.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.352+0000 7ff82c334640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7ff8241363c0 con 0x7ff82412dc90 2024-09-21T01:46:39.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.352+0000 7ff82c334640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7ff824136240 con 0x7ff82412e0a0 2024-09-21T01:46:39.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.352+0000 7ff82c334640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7ff824136780 con 0x7ff824073a70 2024-09-21T01:46:39.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.352+0000 7ff8298cf640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff82412dc90 0x7ff824134be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:39.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.352+0000 7ff82a8d1640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff82412e0a0 0x7ff8241c3da0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:39.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.352+0000 7ff82a0d0640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff824073a70 0x7ff8241346a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:39.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.352+0000 7ff8298cf640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff82412dc90 0x7ff824134be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:52460/0 (socket says 172.21.15.105:52460) 2024-09-21T01:46:39.355 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.352+0000 7ff8298cf640 1 -- 172.21.15.105:0/3230487327 learned_addr learned my addr 172.21.15.105:0/3230487327 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:39.356 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.352+0000 7ff8298cf640 1 -- 172.21.15.105:0/3230487327 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff824073a70 msgr2=0x7ff8241346a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:39.356 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.352+0000 7ff8298cf640 1 --2- 172.21.15.105:0/3230487327 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff824073a70 0x7ff8241346a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.356 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.352+0000 7ff8298cf640 1 -- 172.21.15.105:0/3230487327 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff82412e0a0 msgr2=0x7ff8241c3da0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:39.356 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.352+0000 7ff8298cf640 1 --2- 172.21.15.105:0/3230487327 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff82412e0a0 0x7ff8241c3da0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.356 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.352+0000 7ff8298cf640 1 -- 172.21.15.105:0/3230487327 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7ff8241335c0 con 0x7ff82412dc90 2024-09-21T01:46:39.356 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.352+0000 7ff82a8d1640 1 --2- 172.21.15.105:0/3230487327 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff82412e0a0 0x7ff8241c3da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:39.356 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.352+0000 7ff8298cf640 1 --2- 172.21.15.105:0/3230487327 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff82412dc90 0x7ff824134be0 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7ff81400e9f0 tx=0x7ff81400eeb0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:39.356 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.352+0000 7ff81affd640 1 -- 172.21.15.105:0/3230487327 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff81400cde0 con 0x7ff82412dc90 2024-09-21T01:46:39.357 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.353+0000 7ff81affd640 1 -- 172.21.15.105:0/3230487327 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7ff814004020 con 0x7ff82412dc90 2024-09-21T01:46:39.357 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.353+0000 7ff82c334640 1 -- 172.21.15.105:0/3230487327 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7ff824135120 con 0x7ff82412dc90 2024-09-21T01:46:39.357 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.353+0000 7ff81affd640 1 -- 172.21.15.105:0/3230487327 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7ff814004450 con 0x7ff82412dc90 2024-09-21T01:46:39.357 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.353+0000 7ff82c334640 1 -- 172.21.15.105:0/3230487327 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7ff8241c44c0 con 0x7ff82412dc90 2024-09-21T01:46:39.357 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.354+0000 7ff82c334640 1 -- 172.21.15.105:0/3230487327 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7ff82412b8c0 con 0x7ff82412dc90 2024-09-21T01:46:39.357 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.354+0000 7ff81affd640 1 -- 172.21.15.105:0/3230487327 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7ff8140035f0 con 0x7ff82412dc90 2024-09-21T01:46:39.357 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.354+0000 7ff81affd640 1 -- 172.21.15.105:0/3230487327 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7ff814069970 con 0x7ff82412dc90 2024-09-21T01:46:39.358 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.358+0000 7ff81affd640 1 -- 172.21.15.105:0/3230487327 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7ff814068a20 con 0x7ff82412dc90 2024-09-21T01:46:39.506 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.506+0000 7ff82c334640 1 -- 172.21.15.105:0/3230487327 --> v1:172.21.15.105:6816/2838005468 -- mgr_command(tid 0: {"prefix": "osd deep-scrub", "who": "3", "target": ["mon-mgr", ""]}) -- 0x7ff824133ae0 con 0x7ff8080514b0 2024-09-21T01:46:39.506 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.506+0000 7ff81affd640 1 -- 172.21.15.105:0/3230487327 <== mgr.14125 v1:172.21.15.105:6816/2838005468 1 ==== mgr_command_reply(tid 0: 0 instructed osd(s) 3 to deep-scrub) ==== 41+0+0 (unknown 3526570807 0 0) 0x7ff820001220 con 0x7ff8080514b0 2024-09-21T01:46:39.507 INFO:teuthology.orchestra.run.smithi105.stderr:instructed osd(s) 3 to deep-scrub 2024-09-21T01:46:39.509 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.508+0000 7ff82c334640 1 -- 172.21.15.105:0/3230487327 >> v1:172.21.15.105:6816/2838005468 conn(0x7ff8080514b0 legacy=0x7ff8080539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:39.509 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.508+0000 7ff82c334640 1 -- 172.21.15.105:0/3230487327 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff82412dc90 msgr2=0x7ff824134be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:39.509 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.508+0000 7ff82c334640 1 --2- 172.21.15.105:0/3230487327 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff82412dc90 0x7ff824134be0 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7ff81400e9f0 tx=0x7ff81400eeb0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.509 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.508+0000 7ff82c334640 1 -- 172.21.15.105:0/3230487327 shutdown_connections 2024-09-21T01:46:39.509 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.508+0000 7ff82c334640 1 --2- 172.21.15.105:0/3230487327 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7ff82412e0a0 0x7ff8241c3da0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.509 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.508+0000 7ff82c334640 1 --2- 172.21.15.105:0/3230487327 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7ff82412dc90 0x7ff824134be0 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.509 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.508+0000 7ff82c334640 1 --2- 172.21.15.105:0/3230487327 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7ff824073a70 0x7ff8241346a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.509 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.508+0000 7ff82c334640 1 -- 172.21.15.105:0/3230487327 >> 172.21.15.105:0/3230487327 conn(0x7ff82409f000 msgr2=0x7ff824129ef0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:39.509 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.508+0000 7ff82c334640 1 -- 172.21.15.105:0/3230487327 shutdown_connections 2024-09-21T01:46:39.509 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.508+0000 7ff82c334640 1 -- 172.21.15.105:0/3230487327 wait complete. 2024-09-21T01:46:39.520 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T01:46:39.630 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.629+0000 7f8ea5deb640 1 -- 172.21.15.105:0/2267389736 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8ea0072d10 msgr2=0x7f8ea0073120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:39.630 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.629+0000 7f8ea5deb640 1 --2- 172.21.15.105:0/2267389736 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8ea0072d10 0x7f8ea0073120 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f8e90005fd0 tx=0x7f8e9002a4b0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.630 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.629+0000 7f8ea5deb640 1 -- 172.21.15.105:0/2267389736 shutdown_connections 2024-09-21T01:46:39.630 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.629+0000 7f8ea5deb640 1 --2- 172.21.15.105:0/2267389736 >> v2:172.21.15.105:3300/0 conn(0x7f8ea0099960 0x7f8ea0135e50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.630 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.629+0000 7f8ea5deb640 1 --2- 172.21.15.105:0/2267389736 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8ea0072d10 0x7f8ea0073120 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.630 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.629+0000 7f8ea5deb640 1 -- 172.21.15.105:0/2267389736 >> 172.21.15.105:0/2267389736 conn(0x7f8ea00a01d0 msgr2=0x7f8ea009c6a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:39.630 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.629+0000 7f8ea5deb640 1 -- 172.21.15.105:0/2267389736 shutdown_connections 2024-09-21T01:46:39.630 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.629+0000 7f8ea5deb640 1 -- 172.21.15.105:0/2267389736 wait complete. 2024-09-21T01:46:39.631 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.630+0000 7f8ea5deb640 1 Processor -- start 2024-09-21T01:46:39.631 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.631+0000 7f8ea5deb640 1 -- start start 2024-09-21T01:46:39.633 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.631+0000 7f8ea5deb640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8ea0072d10 0x7f8ea01341d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:39.633 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.631+0000 7f8ea5deb640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8ea0073660 0x7f8ea0134710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:39.633 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.631+0000 7f8ea5deb640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8ea0099960 0x7f8ea0134c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:39.633 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.631+0000 7f8ea5deb640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f8ea0136810 con 0x7f8ea0073660 2024-09-21T01:46:39.633 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.631+0000 7f8ea5deb640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f8ea0136690 con 0x7f8ea0099960 2024-09-21T01:46:39.633 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.631+0000 7f8ea5deb640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f8ea0136990 con 0x7f8ea0072d10 2024-09-21T01:46:39.633 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.631+0000 7f8e9f7fe640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8ea0072d10 0x7f8ea01341d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:39.633 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.631+0000 7f8e9effd640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8ea0073660 0x7f8ea0134710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:39.633 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.631+0000 7f8e9f7fe640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8ea0072d10 0x7f8ea01341d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:48786/0 (socket says 172.21.15.105:48786) 2024-09-21T01:46:39.633 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.631+0000 7f8e9f7fe640 1 -- 172.21.15.105:0/1600251916 learned_addr learned my addr 172.21.15.105:0/1600251916 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:39.633 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.631+0000 7f8e9ffff640 1 --2- 172.21.15.105:0/1600251916 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8ea0099960 0x7f8ea0134c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:39.634 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.632+0000 7f8e9f7fe640 1 -- 172.21.15.105:0/1600251916 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8ea0099960 msgr2=0x7f8ea0134c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:39.634 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.632+0000 7f8e9f7fe640 1 --2- 172.21.15.105:0/1600251916 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8ea0099960 0x7f8ea0134c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.634 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.632+0000 7f8e9f7fe640 1 -- 172.21.15.105:0/1600251916 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8ea0073660 msgr2=0x7f8ea0134710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:39.634 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.632+0000 7f8e9f7fe640 1 --2- 172.21.15.105:0/1600251916 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8ea0073660 0x7f8ea0134710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.634 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.632+0000 7f8e9f7fe640 1 -- 172.21.15.105:0/1600251916 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f8ea0133620 con 0x7f8ea0072d10 2024-09-21T01:46:39.634 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.632+0000 7f8e9effd640 1 --2- 172.21.15.105:0/1600251916 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8ea0073660 0x7f8ea0134710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:46:39.634 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.632+0000 7f8e9f7fe640 1 --2- 172.21.15.105:0/1600251916 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8ea0072d10 0x7f8ea01341d0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f8e90006010 tx=0x7f8e90005e80 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:39.634 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.632+0000 7f8e87fff640 1 -- 172.21.15.105:0/1600251916 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8e9003e030 con 0x7f8ea0072d10 2024-09-21T01:46:39.634 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.632+0000 7f8e87fff640 1 -- 172.21.15.105:0/1600251916 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f8e90003c30 con 0x7f8ea0072d10 2024-09-21T01:46:39.634 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.632+0000 7f8ea5deb640 1 -- 172.21.15.105:0/1600251916 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f8ea01c9c90 con 0x7f8ea0072d10 2024-09-21T01:46:39.634 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.632+0000 7f8e87fff640 1 -- 172.21.15.105:0/1600251916 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f8e90035040 con 0x7f8ea0072d10 2024-09-21T01:46:39.634 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.632+0000 7f8ea5deb640 1 -- 172.21.15.105:0/1600251916 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f8ea01c9e60 con 0x7f8ea0072d10 2024-09-21T01:46:39.635 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.633+0000 7f8ea5deb640 1 -- 172.21.15.105:0/1600251916 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f8ea009b430 con 0x7f8ea0072d10 2024-09-21T01:46:39.635 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.633+0000 7f8e87fff640 1 -- 172.21.15.105:0/1600251916 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f8e90041030 con 0x7f8ea0072d10 2024-09-21T01:46:39.635 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.634+0000 7f8e87fff640 1 -- 172.21.15.105:0/1600251916 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f8e90089690 con 0x7f8ea0072d10 2024-09-21T01:46:39.637 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.637+0000 7f8e87fff640 1 -- 172.21.15.105:0/1600251916 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f8ea009b430 con 0x7f8ea0072d10 2024-09-21T01:46:39.785 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.785+0000 7f8ea5deb640 1 -- 172.21.15.105:0/1600251916 --> v1:172.21.15.105:6816/2838005468 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7f8ea0073120 con 0x7f8e8c0514b0 2024-09-21T01:46:39.786 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.786+0000 7f8e87fff640 1 -- 172.21.15.105:0/1600251916 <== mgr.14125 v1:172.21.15.105:6816/2838005468 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32001 (unknown 2965378022 0 2122761203) 0x7f8ea0073120 con 0x7f8e8c0514b0 2024-09-21T01:46:39.786 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:46:39.787 INFO:teuthology.orchestra.run.smithi105.stderr:dumped all 2024-09-21T01:46:39.788 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.787+0000 7f8ea5deb640 1 -- 172.21.15.105:0/1600251916 >> v1:172.21.15.105:6816/2838005468 conn(0x7f8e8c0514b0 legacy=0x7f8e8c0539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:39.788 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.787+0000 7f8ea5deb640 1 -- 172.21.15.105:0/1600251916 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8ea0072d10 msgr2=0x7f8ea01341d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:39.788 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.787+0000 7f8ea5deb640 1 --2- 172.21.15.105:0/1600251916 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8ea0072d10 0x7f8ea01341d0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f8e90006010 tx=0x7f8e90005e80 comp rx=0 tx=0).stop 2024-09-21T01:46:39.788 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.788+0000 7f8ea5deb640 1 -- 172.21.15.105:0/1600251916 shutdown_connections 2024-09-21T01:46:39.788 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.788+0000 7f8ea5deb640 1 --2- 172.21.15.105:0/1600251916 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f8ea0099960 0x7f8ea0134c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.788 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.788+0000 7f8ea5deb640 1 --2- 172.21.15.105:0/1600251916 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f8ea0073660 0x7f8ea0134710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.788 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.788+0000 7f8ea5deb640 1 --2- 172.21.15.105:0/1600251916 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f8ea0072d10 0x7f8ea01341d0 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:39.788 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.788+0000 7f8ea5deb640 1 -- 172.21.15.105:0/1600251916 >> 172.21.15.105:0/1600251916 conn(0x7f8ea00a01d0 msgr2=0x7f8ea012ccf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:39.789 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.788+0000 7f8ea5deb640 1 -- 172.21.15.105:0/1600251916 shutdown_connections 2024-09-21T01:46:39.789 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:39.788+0000 7f8ea5deb640 1 -- 172.21.15.105:0/1600251916 wait complete. 2024-09-21T01:46:39.799 INFO:teuthology.orchestra.run.smithi105.stdout:{"pg_ready":true,"pg_map":{"version":29,"stamp":"2024-09-21T01:46:39.502327+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107,"ondisk_log_size":107,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112348,"kb_used_data":2256,"kb_used_omap":17,"kb_used_meta":109998,"kb_avail":377375012,"statfs":{"total":386547056640,"available":386432012288,"internally_reserved":0,"allocated":2310144,"data_stored":1569618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17745,"internal_metadata":112638639},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.854469"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":49,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064136+0000","last_change":"2024-09-21T01:46:28.651690+0000","last_active":"2024-09-21T01:46:32.064136+0000","last_peered":"2024-09-21T01:46:32.064136+0000","last_clean":"2024-09-21T01:46:32.064136+0000","last_became_active":"2024-09-21T01:46:28.651383+0000","last_became_peered":"2024-09-21T01:46:28.651383+0000","last_unstale":"2024-09-21T01:46:32.064136+0000","last_undegraded":"2024-09-21T01:46:32.064136+0000","last_fullsized":"2024-09-21T01:46:32.064136+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T01:49:28.532164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040868700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":39,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657657+0000","last_change":"2024-09-21T01:46:26.623500+0000","last_active":"2024-09-21T01:46:31.657657+0000","last_peered":"2024-09-21T01:46:31.657657+0000","last_clean":"2024-09-21T01:46:31.657657+0000","last_became_active":"2024-09-21T01:46:26.623142+0000","last_became_peered":"2024-09-21T01:46:26.623142+0000","last_unstale":"2024-09-21T01:46:31.657657+0000","last_undegraded":"2024-09-21T01:46:31.657657+0000","last_fullsized":"2024-09-21T01:46:31.657657+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:05:55.215381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055542700000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":49,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064256+0000","last_change":"2024-09-21T01:46:28.649216+0000","last_active":"2024-09-21T01:46:32.064256+0000","last_peered":"2024-09-21T01:46:32.064256+0000","last_clean":"2024-09-21T01:46:32.064256+0000","last_became_active":"2024-09-21T01:46:28.648886+0000","last_became_peered":"2024-09-21T01:46:28.648886+0000","last_unstale":"2024-09-21T01:46:32.064256+0000","last_undegraded":"2024-09-21T01:46:32.064256+0000","last_fullsized":"2024-09-21T01:46:32.064256+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:09:06.681843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00119484,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":39,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657839+0000","last_change":"2024-09-21T01:46:26.624399+0000","last_active":"2024-09-21T01:46:31.657839+0000","last_peered":"2024-09-21T01:46:31.657839+0000","last_clean":"2024-09-21T01:46:31.657839+0000","last_became_active":"2024-09-21T01:46:26.624073+0000","last_became_peered":"2024-09-21T01:46:26.624073+0000","last_unstale":"2024-09-21T01:46:31.657839+0000","last_undegraded":"2024-09-21T01:46:31.657839+0000","last_fullsized":"2024-09-21T01:46:31.657839+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T08:18:28.384798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063215099999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":46,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064137+0000","last_change":"2024-09-21T01:46:28.654323+0000","last_active":"2024-09-21T01:46:32.064137+0000","last_peered":"2024-09-21T01:46:32.064137+0000","last_clean":"2024-09-21T01:46:32.064137+0000","last_became_active":"2024-09-21T01:46:28.653984+0000","last_became_peered":"2024-09-21T01:46:28.653984+0000","last_unstale":"2024-09-21T01:46:32.064137+0000","last_undegraded":"2024-09-21T01:46:32.064137+0000","last_fullsized":"2024-09-21T01:46:32.064137+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:41:46.114559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010245149999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":36,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657473+0000","last_change":"2024-09-21T01:46:28.656954+0000","last_active":"2024-09-21T01:46:31.657473+0000","last_peered":"2024-09-21T01:46:31.657473+0000","last_clean":"2024-09-21T01:46:31.657473+0000","last_became_active":"2024-09-21T01:46:28.656588+0000","last_became_peered":"2024-09-21T01:46:28.656588+0000","last_unstale":"2024-09-21T01:46:31.657473+0000","last_undegraded":"2024-09-21T01:46:31.657473+0000","last_fullsized":"2024-09-21T01:46:31.657473+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T02:10:00.211563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080666500000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":44,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064104+0000","last_change":"2024-09-21T01:46:28.657038+0000","last_active":"2024-09-21T01:46:32.064104+0000","last_peered":"2024-09-21T01:46:32.064104+0000","last_clean":"2024-09-21T01:46:32.064104+0000","last_became_active":"2024-09-21T01:46:28.656701+0000","last_became_peered":"2024-09-21T01:46:28.656701+0000","last_unstale":"2024-09-21T01:46:32.064104+0000","last_undegraded":"2024-09-21T01:46:32.064104+0000","last_fullsized":"2024-09-21T01:46:32.064104+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T05:11:34.297379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058759600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":42,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:32.064133+0000","last_change":"2024-09-21T01:46:28.657189+0000","last_active":"2024-09-21T01:46:32.064133+0000","last_peered":"2024-09-21T01:46:32.064133+0000","last_clean":"2024-09-21T01:46:32.064133+0000","last_became_active":"2024-09-21T01:46:28.656728+0000","last_became_peered":"2024-09-21T01:46:28.656728+0000","last_unstale":"2024-09-21T01:46:32.064133+0000","last_undegraded":"2024-09-21T01:46:32.064133+0000","last_fullsized":"2024-09-21T01:46:32.064133+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:54.506412+0000","last_clean_scrub_stamp":"2024-09-21T01:44:54.506412+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T08:49:15.772867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036256899999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"20'104","reported_seq":189,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:31.657393+0000","last_change":"2024-09-21T01:46:26.626044+0000","last_active":"2024-09-21T01:46:31.657393+0000","last_peered":"2024-09-21T01:46:31.657393+0000","last_clean":"2024-09-21T01:46:31.657393+0000","last_became_active":"2024-09-21T01:46:26.625434+0000","last_became_peered":"2024-09-21T01:46:26.625434+0000","last_unstale":"2024-09-21T01:46:31.657393+0000","last_undegraded":"2024-09-21T01:46:31.657393+0000","last_fullsized":"2024-09-21T01:46:31.657393+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:44:50.489745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:44:50.489745+0000","last_clean_scrub_stamp":"2024-09-21T01:44:50.489745+0000","objects_scrubbed":0,"log_size":104,"log_dups_size":0,"ondisk_log_size":104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T08:39:55.782994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104,"ondisk_log_size":104,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":29,"seq":124554051588,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27816,"kb_used_data":276,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344024,"statfs":{"total":96636764160,"available":96608280576,"internally_reserved":0,"allocated":282624,"data_stored":97230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4435,"internal_metadata":28176045},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":29,"seq":124554051588,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27812,"kb_used_data":272,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344028,"statfs":{"total":96636764160,"available":96608284672,"internally_reserved":0,"allocated":278528,"data_stored":97211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28176043},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":27,"seq":115964116996,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28328,"kb_used_data":856,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343512,"statfs":{"total":96636764160,"available":96607756288,"internally_reserved":0,"allocated":876544,"data_stored":687598,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4436,"internal_metadata":28110508},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":26,"seq":111669149701,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28392,"kb_used_data":852,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94343448,"statfs":{"total":96636764160,"available":96607690752,"internally_reserved":0,"allocated":872448,"data_stored":687579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28176043},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T01:46:39.802 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2024-09-21T01:44:54.506412+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=1, tm_min=44, tm_sec=54, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=1, tm_min=46, tm_sec=36, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T01:46:39.802 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2024-09-21T01:44:54.506412+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=1, tm_min=44, tm_sec=54, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=1, tm_min=46, tm_sec=36, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T01:46:39.802 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2024-09-21T01:44:54.506412+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=1, tm_min=44, tm_sec=54, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=1, tm_min=46, tm_sec=36, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T01:46:39.802 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2024-09-21T01:44:54.506412+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=1, tm_min=44, tm_sec=54, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=1, tm_min=46, tm_sec=36, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T01:46:39.802 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2024-09-21T01:44:54.506412+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=1, tm_min=44, tm_sec=54, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=1, tm_min=46, tm_sec=36, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T01:46:39.802 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2024-09-21T01:44:54.506412+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=1, tm_min=44, tm_sec=54, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=1, tm_min=46, tm_sec=36, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T01:46:39.802 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2024-09-21T01:44:54.506412+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=1, tm_min=44, tm_sec=54, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=1, tm_min=46, tm_sec=36, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T01:46:39.802 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2024-09-21T01:44:54.506412+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=1, tm_min=44, tm_sec=54, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=1, tm_min=46, tm_sec=36, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T01:46:39.803 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2024-09-21T01:44:50.489745+0000 time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=1, tm_min=44, tm_sec=50, tm_wday=5, tm_yday=265, tm_isdst=-1) <= time.struct_time(tm_year=2024, tm_mon=9, tm_mday=21, tm_hour=1, tm_min=46, tm_sec=36, tm_wday=5, tm_yday=265, tm_isdst=0) 2024-09-21T01:46:39.803 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2024-09-21T01:46:59.804 DEBUG:teuthology.orchestra.run.smithi105:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json 2024-09-21T01:46:59.929 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.927+0000 7fb1b2861640 1 -- 172.21.15.105:0/1780566276 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb1ac096b70 msgr2=0x7fb1ac096f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:59.929 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.927+0000 7fb1b2861640 1 --2- 172.21.15.105:0/1780566276 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb1ac096b70 0x7fb1ac096f80 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7fb19c005fd0 tx=0x7fb19c02a410 comp rx=0 tx=0).stop 2024-09-21T01:46:59.929 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.927+0000 7fb1b2861640 1 -- 172.21.15.105:0/1780566276 shutdown_connections 2024-09-21T01:46:59.929 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.927+0000 7fb1b2861640 1 --2- 172.21.15.105:0/1780566276 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb1ac0945b0 0x7fb1ac0949c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:59.929 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.927+0000 7fb1b2861640 1 --2- 172.21.15.105:0/1780566276 >> v2:172.21.15.105:3300/0 conn(0x7fb1ac0974c0 0x7fb1ac094070 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:59.930 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.927+0000 7fb1b2861640 1 --2- 172.21.15.105:0/1780566276 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb1ac096b70 0x7fb1ac096f80 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:59.930 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.927+0000 7fb1b2861640 1 -- 172.21.15.105:0/1780566276 >> 172.21.15.105:0/1780566276 conn(0x7fb1ac09e4e0 msgr2=0x7fb1ac09e8f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:46:59.930 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.927+0000 7fb1b2861640 1 -- 172.21.15.105:0/1780566276 shutdown_connections 2024-09-21T01:46:59.930 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.927+0000 7fb1b2861640 1 -- 172.21.15.105:0/1780566276 wait complete. 2024-09-21T01:46:59.932 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.928+0000 7fb1b2861640 1 Processor -- start 2024-09-21T01:46:59.932 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.929+0000 7fb1b2861640 1 -- start start 2024-09-21T01:46:59.932 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.929+0000 7fb1b2861640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb1ac0945b0 0x7fb1ac1c2630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:59.932 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.929+0000 7fb1b2861640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb1ac096b70 0x7fb1ac1c2b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:59.932 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.929+0000 7fb1b2861640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb1ac0974c0 0x7fb1ac1c30b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:46:59.932 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.929+0000 7fb1b2861640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7fb1ac1341d0 con 0x7fb1ac0974c0 2024-09-21T01:46:59.932 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.929+0000 7fb1b2861640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7fb1ac134050 con 0x7fb1ac0945b0 2024-09-21T01:46:59.932 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.929+0000 7fb1b2861640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7fb1ac134350 con 0x7fb1ac096b70 2024-09-21T01:46:59.932 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.929+0000 7fb1abfff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb1ac0945b0 0x7fb1ac1c2630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:59.932 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.929+0000 7fb1ab7fe640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb1ac096b70 0x7fb1ac1c2b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:59.933 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.929+0000 7fb1b0dfe640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb1ac0974c0 0x7fb1ac1c30b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:46:59.933 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.929+0000 7fb1abfff640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb1ac0945b0 0x7fb1ac1c2630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3301/0 says I am v2:172.21.15.105:43940/0 (socket says 172.21.15.105:43940) 2024-09-21T01:46:59.933 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.929+0000 7fb1ab7fe640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb1ac096b70 0x7fb1ac1c2b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:39938/0 (socket says 172.21.15.105:39938) 2024-09-21T01:46:59.933 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.929+0000 7fb1abfff640 1 -- 172.21.15.105:0/3959361281 learned_addr learned my addr 172.21.15.105:0/3959361281 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:46:59.933 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.930+0000 7fb1b0dfe640 1 -- 172.21.15.105:0/3959361281 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb1ac096b70 msgr2=0x7fb1ac1c2b70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:59.933 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.930+0000 7fb1b0dfe640 1 --2- 172.21.15.105:0/3959361281 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb1ac096b70 0x7fb1ac1c2b70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:59.933 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.930+0000 7fb1b0dfe640 1 -- 172.21.15.105:0/3959361281 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb1ac0945b0 msgr2=0x7fb1ac1c2630 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:46:59.933 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.930+0000 7fb1b0dfe640 1 --2- 172.21.15.105:0/3959361281 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb1ac0945b0 0x7fb1ac1c2630 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:46:59.933 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.930+0000 7fb1b0dfe640 1 -- 172.21.15.105:0/3959361281 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7fb1ac124160 con 0x7fb1ac0974c0 2024-09-21T01:46:59.933 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.930+0000 7fb1b0dfe640 1 --2- 172.21.15.105:0/3959361281 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb1ac0974c0 0x7fb1ac1c30b0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7fb1a4005fd0 tx=0x7fb1a400edd0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:46:59.933 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.930+0000 7fb1a8ff9640 1 -- 172.21.15.105:0/3959361281 <== mon.0 v2:172.21.15.105:3300/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb1a400cd80 con 0x7fb1ac0974c0 2024-09-21T01:46:59.933 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.930+0000 7fb1a8ff9640 1 -- 172.21.15.105:0/3959361281 <== mon.0 v2:172.21.15.105:3300/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7fb1a4004b20 con 0x7fb1ac0974c0 2024-09-21T01:46:59.933 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.930+0000 7fb1b2861640 1 -- 172.21.15.105:0/3959361281 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({mgrmap=0+}) -- 0x7fb1ac132070 con 0x7fb1ac0974c0 2024-09-21T01:46:59.934 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.930+0000 7fb1a8ff9640 1 -- 172.21.15.105:0/3959361281 <== mon.0 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7fb1a4005020 con 0x7fb1ac0974c0 2024-09-21T01:46:59.934 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.930+0000 7fb1b2861640 1 -- 172.21.15.105:0/3959361281 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_subscribe({osdmap=0}) -- 0x7fb1ac132570 con 0x7fb1ac0974c0 2024-09-21T01:46:59.934 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.931+0000 7fb1b2861640 1 -- 172.21.15.105:0/3959361281 --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7fb1ac096f80 con 0x7fb1ac0974c0 2024-09-21T01:46:59.934 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.932+0000 7fb1a8ff9640 1 -- 172.21.15.105:0/3959361281 <== mon.0 v2:172.21.15.105:3300/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7fb1a40051c0 con 0x7fb1ac0974c0 2024-09-21T01:46:59.934 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.932+0000 7fb1a8ff9640 1 -- 172.21.15.105:0/3959361281 <== mon.0 v2:172.21.15.105:3300/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7fb1a4069b20 con 0x7fb1ac0974c0 2024-09-21T01:46:59.937 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:46:59.936+0000 7fb1a8ff9640 1 -- 172.21.15.105:0/3959361281 <== mon.0 v2:172.21.15.105:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7fb1a4068bd0 con 0x7fb1ac0974c0 2024-09-21T01:47:00.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.079+0000 7fb1b2861640 1 -- 172.21.15.105:0/3959361281 --> v1:172.21.15.105:6816/2838005468 -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) -- 0x7fb1ac133200 con 0x7fb1980514b0 2024-09-21T01:47:00.081 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.080+0000 7fb1a8ff9640 1 -- 172.21.15.105:0/3959361281 <== mgr.14125 v1:172.21.15.105:6816/2838005468 1 ==== mgr_command_reply(tid 0: 0 dumped all) ==== 18+0+32012 (unknown 2965378022 0 530210517) 0x7fb1ac133200 con 0x7fb1980514b0 2024-09-21T01:47:00.082 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T01:47:00.082 INFO:teuthology.orchestra.run.smithi105.stderr:dumped all 2024-09-21T01:47:00.084 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.083+0000 7fb1b2861640 1 -- 172.21.15.105:0/3959361281 >> v1:172.21.15.105:6816/2838005468 conn(0x7fb1980514b0 legacy=0x7fb1980539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:47:00.084 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.083+0000 7fb1b2861640 1 -- 172.21.15.105:0/3959361281 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb1ac0974c0 msgr2=0x7fb1ac1c30b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:47:00.084 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.083+0000 7fb1b2861640 1 --2- 172.21.15.105:0/3959361281 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb1ac0974c0 0x7fb1ac1c30b0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7fb1a4005fd0 tx=0x7fb1a400edd0 comp rx=0 tx=0).stop 2024-09-21T01:47:00.084 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.083+0000 7fb1b2861640 1 -- 172.21.15.105:0/3959361281 shutdown_connections 2024-09-21T01:47:00.084 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.083+0000 7fb1b2861640 1 --2- 172.21.15.105:0/3959361281 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7fb1ac0974c0 0x7fb1ac1c30b0 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:47:00.084 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.083+0000 7fb1b2861640 1 --2- 172.21.15.105:0/3959361281 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7fb1ac096b70 0x7fb1ac1c2b70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:47:00.085 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.083+0000 7fb1b2861640 1 --2- 172.21.15.105:0/3959361281 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7fb1ac0945b0 0x7fb1ac1c2630 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:47:00.085 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.083+0000 7fb1b2861640 1 -- 172.21.15.105:0/3959361281 >> 172.21.15.105:0/3959361281 conn(0x7fb1ac09e4e0 msgr2=0x7fb1ac099b50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:47:00.085 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.083+0000 7fb1b2861640 1 -- 172.21.15.105:0/3959361281 shutdown_connections 2024-09-21T01:47:00.085 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.083+0000 7fb1b2861640 1 -- 172.21.15.105:0/3959361281 wait complete. 2024-09-21T01:47:00.095 INFO:teuthology.orchestra.run.smithi105.stdout:{"pg_ready":true,"pg_map":{"version":39,"stamp":"2024-09-21T01:46:59.506130+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":151,"num_write_kb":1896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110,"ondisk_log_size":110,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":377487360,"kb_used":112304,"kb_used_data":2256,"kb_used_omap":17,"kb_used_meta":109998,"kb_avail":377375056,"statfs":{"total":386547056640,"available":386432057344,"internally_reserved":0,"allocated":2310144,"data_stored":1569618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17745,"internal_metadata":112638639},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002242"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":57,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:41.131339+0000","last_change":"2024-09-21T01:46:41.131339+0000","last_active":"2024-09-21T01:46:41.131339+0000","last_peered":"2024-09-21T01:46:41.131339+0000","last_clean":"2024-09-21T01:46:41.131339+0000","last_became_active":"2024-09-21T01:46:28.651383+0000","last_became_peered":"2024-09-21T01:46:28.651383+0000","last_unstale":"2024-09-21T01:46:41.131339+0000","last_undegraded":"2024-09-21T01:46:41.131339+0000","last_fullsized":"2024-09-21T01:46:41.131339+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:46:41.131229+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:46:41.131229+0000","last_clean_scrub_stamp":"2024-09-21T01:46:41.131229+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T04:03:24.910332+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00040868700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":47,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:41.244315+0000","last_change":"2024-09-21T01:46:41.244315+0000","last_active":"2024-09-21T01:46:41.244315+0000","last_peered":"2024-09-21T01:46:41.244315+0000","last_clean":"2024-09-21T01:46:41.244315+0000","last_became_active":"2024-09-21T01:46:26.623142+0000","last_became_peered":"2024-09-21T01:46:26.623142+0000","last_unstale":"2024-09-21T01:46:41.244315+0000","last_undegraded":"2024-09-21T01:46:41.244315+0000","last_fullsized":"2024-09-21T01:46:41.244315+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:46:41.244215+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:46:41.244215+0000","last_clean_scrub_stamp":"2024-09-21T01:46:41.244215+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:18:59.133370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055542700000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":57,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:43.117592+0000","last_change":"2024-09-21T01:46:43.117592+0000","last_active":"2024-09-21T01:46:43.117592+0000","last_peered":"2024-09-21T01:46:43.117592+0000","last_clean":"2024-09-21T01:46:43.117592+0000","last_became_active":"2024-09-21T01:46:28.648886+0000","last_became_peered":"2024-09-21T01:46:28.648886+0000","last_unstale":"2024-09-21T01:46:43.117592+0000","last_undegraded":"2024-09-21T01:46:43.117592+0000","last_fullsized":"2024-09-21T01:46:43.117592+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:46:43.117488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:46:43.117488+0000","last_clean_scrub_stamp":"2024-09-21T01:46:43.117488+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:41:09.384617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00119484,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":47,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:40.250595+0000","last_change":"2024-09-21T01:46:40.250595+0000","last_active":"2024-09-21T01:46:40.250595+0000","last_peered":"2024-09-21T01:46:40.250595+0000","last_clean":"2024-09-21T01:46:40.250595+0000","last_became_active":"2024-09-21T01:46:26.624073+0000","last_became_peered":"2024-09-21T01:46:26.624073+0000","last_unstale":"2024-09-21T01:46:40.250595+0000","last_undegraded":"2024-09-21T01:46:40.250595+0000","last_fullsized":"2024-09-21T01:46:40.250595+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:46:40.250514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:46:40.250514+0000","last_clean_scrub_stamp":"2024-09-21T01:46:40.250514+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T03:46:27.598135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00063215099999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"33'3","reported_seq":55,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:42.134271+0000","last_change":"2024-09-21T01:46:42.134271+0000","last_active":"2024-09-21T01:46:42.134271+0000","last_peered":"2024-09-21T01:46:42.134271+0000","last_clean":"2024-09-21T01:46:42.134271+0000","last_became_active":"2024-09-21T01:46:28.653984+0000","last_became_peered":"2024-09-21T01:46:28.653984+0000","last_unstale":"2024-09-21T01:46:42.134271+0000","last_undegraded":"2024-09-21T01:46:42.134271+0000","last_fullsized":"2024-09-21T01:46:42.134271+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"33'3","last_scrub_stamp":"2024-09-21T01:46:42.134175+0000","last_deep_scrub":"33'3","last_deep_scrub_stamp":"2024-09-21T01:46:42.134175+0000","last_clean_scrub_stamp":"2024-09-21T01:46:42.134175+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T07:33:55.708187+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0.0010245149999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":44,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:40.041329+0000","last_change":"2024-09-21T01:46:40.041329+0000","last_active":"2024-09-21T01:46:40.041329+0000","last_peered":"2024-09-21T01:46:40.041329+0000","last_clean":"2024-09-21T01:46:40.041329+0000","last_became_active":"2024-09-21T01:46:28.656588+0000","last_became_peered":"2024-09-21T01:46:28.656588+0000","last_unstale":"2024-09-21T01:46:40.041329+0000","last_undegraded":"2024-09-21T01:46:40.041329+0000","last_fullsized":"2024-09-21T01:46:40.041329+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:46:40.041240+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:46:40.041240+0000","last_clean_scrub_stamp":"2024-09-21T01:46:40.041240+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T06:53:30.193896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080666500000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":52,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:40.171758+0000","last_change":"2024-09-21T01:46:40.171758+0000","last_active":"2024-09-21T01:46:40.171758+0000","last_peered":"2024-09-21T01:46:40.171758+0000","last_clean":"2024-09-21T01:46:40.171758+0000","last_became_active":"2024-09-21T01:46:28.656701+0000","last_became_peered":"2024-09-21T01:46:28.656701+0000","last_unstale":"2024-09-21T01:46:40.171758+0000","last_undegraded":"2024-09-21T01:46:40.171758+0000","last_fullsized":"2024-09-21T01:46:40.171758+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T01:46:40.171679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T01:46:40.171679+0000","last_clean_scrub_stamp":"2024-09-21T01:46:40.171679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T05:32:22.628325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058759600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"33'2","reported_seq":51,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:44.083081+0000","last_change":"2024-09-21T01:46:44.083081+0000","last_active":"2024-09-21T01:46:44.083081+0000","last_peered":"2024-09-21T01:46:44.083081+0000","last_clean":"2024-09-21T01:46:44.083081+0000","last_became_active":"2024-09-21T01:46:28.656728+0000","last_became_peered":"2024-09-21T01:46:28.656728+0000","last_unstale":"2024-09-21T01:46:44.083081+0000","last_undegraded":"2024-09-21T01:46:44.083081+0000","last_fullsized":"2024-09-21T01:46:44.083081+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"33'2","last_scrub_stamp":"2024-09-21T01:46:44.082959+0000","last_deep_scrub":"33'2","last_deep_scrub_stamp":"2024-09-21T01:46:44.082959+0000","last_clean_scrub_stamp":"2024-09-21T01:46:44.082959+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:27:52.248937+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0.00036256899999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"33'105","reported_seq":198,"reported_epoch":33,"state":"active+clean","last_fresh":"2024-09-21T01:46:39.276452+0000","last_change":"2024-09-21T01:46:39.276452+0000","last_active":"2024-09-21T01:46:39.276452+0000","last_peered":"2024-09-21T01:46:39.276452+0000","last_clean":"2024-09-21T01:46:39.276452+0000","last_became_active":"2024-09-21T01:46:26.625434+0000","last_became_peered":"2024-09-21T01:46:26.625434+0000","last_unstale":"2024-09-21T01:46:39.276452+0000","last_undegraded":"2024-09-21T01:46:39.276452+0000","last_fullsized":"2024-09-21T01:46:39.276452+0000","mapping_epoch":27,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":28,"parent":"0.0","parent_split_bits":0,"last_scrub":"33'105","last_scrub_stamp":"2024-09-21T01:46:39.276362+0000","last_deep_scrub":"33'105","last_deep_scrub_stamp":"2024-09-21T01:46:39.276362+0000","last_clean_scrub_stamp":"2024-09-21T01:46:39.276362+0000","objects_scrubbed":2,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:37:50.470784+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":166,"num_read_kb":267,"num_write":149,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105,"ondisk_log_size":105,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":3,"up_from":29,"seq":124554051592,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27804,"kb_used_data":276,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344036,"statfs":{"total":96636764160,"available":96608292864,"internally_reserved":0,"allocated":282624,"data_stored":97230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4435,"internal_metadata":28176045},"hb_peers":[0,1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":29,"seq":124554051592,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27800,"kb_used_data":272,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94344040,"statfs":{"total":96636764160,"available":96608296960,"internally_reserved":0,"allocated":278528,"data_stored":97211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28176043},"hb_peers":[0,1,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":27,"seq":115964117000,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28320,"kb_used_data":856,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":94343520,"statfs":{"total":96636764160,"available":96607764480,"internally_reserved":0,"allocated":876544,"data_stored":687598,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4436,"internal_metadata":28110508},"hb_peers":[0,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":26,"seq":111669149705,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28380,"kb_used_data":852,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":94343460,"statfs":{"total":96636764160,"available":96607703040,"internally_reserved":0,"allocated":872448,"data_stored":687579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4437,"internal_metadata":28176043},"hb_peers":[1,2,3],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T01:47:00.096 DEBUG:teuthology.orchestra.run.smithi105:> sudo ceph --cluster ceph config set global mon_health_to_clog false 2024-09-21T01:47:00.207 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.205+0000 7f82b4c0c640 1 -- 172.21.15.105:0/2107041190 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f82b0135190 msgr2=0x7f82b01375b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:47:00.207 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.205+0000 7f82b4c0c640 1 --2- 172.21.15.105:0/2107041190 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f82b0135190 0x7f82b01375b0 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f82a8005fd0 tx=0x7f82a802a780 comp rx=0 tx=0).stop 2024-09-21T01:47:00.207 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.205+0000 7f82b4c0c640 1 -- 172.21.15.105:0/2107041190 shutdown_connections 2024-09-21T01:47:00.207 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.205+0000 7f82b4c0c640 1 --2- 172.21.15.105:0/2107041190 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f82b0135190 0x7f82b01375b0 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:47:00.207 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.205+0000 7f82b4c0c640 1 --2- 172.21.15.105:0/2107041190 >> v2:172.21.15.105:3300/0 conn(0x7f82b0130170 0x7f82b0134c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:47:00.207 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.205+0000 7f82b4c0c640 1 -- 172.21.15.105:0/2107041190 >> 172.21.15.105:0/2107041190 conn(0x7f82b00a1e40 msgr2=0x7f82b00a2250 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:47:00.207 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.205+0000 7f82b4c0c640 1 -- 172.21.15.105:0/2107041190 shutdown_connections 2024-09-21T01:47:00.207 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.205+0000 7f82b4c0c640 1 -- 172.21.15.105:0/2107041190 wait complete. 2024-09-21T01:47:00.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.206+0000 7f82b4c0c640 1 Processor -- start 2024-09-21T01:47:00.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.206+0000 7f82b4c0c640 1 -- start start 2024-09-21T01:47:00.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.207+0000 7f82b4c0c640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f82b012f650 0x7f82b009b4b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:47:00.209 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.207+0000 7f82b4c0c640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f82b0130170 0x7f82b009b9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:47:00.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.207+0000 7f82b4c0c640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f82b0135190 0x7f82b009bf30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-09-21T01:47:00.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.207+0000 7f82b4c0c640 1 -- --> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] -- mon_getmap magic: 0 -- 0x7f82b0138690 con 0x7f82b0130170 2024-09-21T01:47:00.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.207+0000 7f82b4c0c640 1 -- --> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] -- mon_getmap magic: 0 -- 0x7f82b0138510 con 0x7f82b0135190 2024-09-21T01:47:00.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.207+0000 7f82b4c0c640 1 -- --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_getmap magic: 0 -- 0x7f82b0138810 con 0x7f82b012f650 2024-09-21T01:47:00.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.207+0000 7f82ae59c640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f82b012f650 0x7f82b009b4b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:47:00.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.207+0000 7f82add9b640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f82b0130170 0x7f82b009b9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:47:00.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.207+0000 7f82aed9d640 1 --2- >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f82b0135190 0x7f82b009bf30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-09-21T01:47:00.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.207+0000 7f82ae59c640 1 --2- >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f82b012f650 0x7f82b009b4b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3302/0 says I am v2:172.21.15.105:39940/0 (socket says 172.21.15.105:39940) 2024-09-21T01:47:00.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.207+0000 7f82ae59c640 1 -- 172.21.15.105:0/3681912882 learned_addr learned my addr 172.21.15.105:0/3681912882 (peer_addr_for_me v2:172.21.15.105:0/0) 2024-09-21T01:47:00.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.207+0000 7f82add9b640 1 --2- >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f82b0130170 0x7f82b009b9f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.105:3300/0 says I am v2:172.21.15.105:40752/0 (socket says 172.21.15.105:40752) 2024-09-21T01:47:00.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.207+0000 7f82ae59c640 1 -- 172.21.15.105:0/3681912882 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f82b0135190 msgr2=0x7f82b009bf30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:47:00.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.207+0000 7f82ae59c640 1 --2- 172.21.15.105:0/3681912882 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f82b0135190 0x7f82b009bf30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:47:00.210 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.207+0000 7f82ae59c640 1 -- 172.21.15.105:0/3681912882 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f82b0130170 msgr2=0x7f82b009b9f0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:47:00.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.207+0000 7f82ae59c640 1 --2- 172.21.15.105:0/3681912882 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f82b0130170 0x7f82b009b9f0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:47:00.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.207+0000 7f82ae59c640 1 -- 172.21.15.105:0/3681912882 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({config=0+,monmap=0+}) -- 0x7f82b0131700 con 0x7f82b012f650 2024-09-21T01:47:00.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.207+0000 7f82add9b640 1 --2- 172.21.15.105:0/3681912882 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f82b0130170 0x7f82b009b9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:47:00.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.207+0000 7f82aed9d640 1 --2- 172.21.15.105:0/3681912882 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f82b0135190 0x7f82b009bf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-09-21T01:47:00.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.208+0000 7f82ae59c640 1 --2- 172.21.15.105:0/3681912882 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f82b012f650 0x7f82b009b4b0 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f82a000c430 tx=0x7f82a000c8f0 comp rx=0 tx=0).ready entity=mon.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-09-21T01:47:00.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.208+0000 7f829effd640 1 -- 172.21.15.105:0/3681912882 <== mon.2 v2:172.21.15.105:3302/0 1 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f82a0011790 con 0x7f82b012f650 2024-09-21T01:47:00.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.208+0000 7f829effd640 1 -- 172.21.15.105:0/3681912882 <== mon.2 v2:172.21.15.105:3302/0 2 ==== config(5 keys) ==== 141+0+0 (secure 0 0 0) 0x7f82a0017070 con 0x7f82b012f650 2024-09-21T01:47:00.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.208+0000 7f82b4c0c640 1 -- 172.21.15.105:0/3681912882 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({mgrmap=0+}) -- 0x7f82b0131000 con 0x7f82b012f650 2024-09-21T01:47:00.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.208+0000 7f829effd640 1 -- 172.21.15.105:0/3681912882 <== mon.2 v2:172.21.15.105:3302/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x7f82a0011ca0 con 0x7f82b012f650 2024-09-21T01:47:00.211 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.208+0000 7f82b4c0c640 1 -- 172.21.15.105:0/3681912882 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_subscribe({osdmap=0}) -- 0x7f82b012b4e0 con 0x7f82b012f650 2024-09-21T01:47:00.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.209+0000 7f829effd640 1 -- 172.21.15.105:0/3681912882 <== mon.2 v2:172.21.15.105:3302/0 4 ==== mgrmap(e 10) ==== 63498+0+0 (secure 0 0 0) 0x7f82a001c460 con 0x7f82b012f650 2024-09-21T01:47:00.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.209+0000 7f82b4c0c640 1 -- 172.21.15.105:0/3681912882 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) -- 0x7f82b012fa60 con 0x7f82b012f650 2024-09-21T01:47:00.212 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.209+0000 7f829effd640 1 -- 172.21.15.105:0/3681912882 <== mon.2 v2:172.21.15.105:3302/0 5 ==== osd_map(33..33 src has 1..33) ==== 3027+0+0 (secure 0 0 0) 0x7f82a0068d40 con 0x7f82b012f650 2024-09-21T01:47:00.214 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.213+0000 7f829effd640 1 -- 172.21.15.105:0/3681912882 <== mon.2 v2:172.21.15.105:3302/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) ==== 72+0+158489 (secure 0 0 0) 0x7f82a0067df0 con 0x7f82b012f650 2024-09-21T01:47:00.363 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.362+0000 7f82b4c0c640 1 -- 172.21.15.105:0/3681912882 --> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] -- mon_command([{prefix=config set, name=mon_health_to_clog}] v 0) -- 0x7f82b0132220 con 0x7f82b012f650 2024-09-21T01:47:00.368 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.366+0000 7f829effd640 1 -- 172.21.15.105:0/3681912882 <== mon.2 v2:172.21.15.105:3302/0 7 ==== config(6 keys) ==== 172+0+0 (secure 0 0 0) 0x7f82a00451d0 con 0x7f82b012f650 2024-09-21T01:47:00.368 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.366+0000 7f829effd640 1 -- 172.21.15.105:0/3681912882 <== mon.2 v2:172.21.15.105:3302/0 8 ==== mon_command_ack([{prefix=config set, name=mon_health_to_clog}]=0 v28)=0 v28) ==== 123+0+0 (secure 0 0 0) 0x7f82a0011e40 con 0x7f82b012f650 2024-09-21T01:47:00.370 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.369+0000 7f82b4c0c640 1 -- 172.21.15.105:0/3681912882 >> v1:172.21.15.105:6816/2838005468 conn(0x7f82900514b0 legacy=0x7f82900539c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:47:00.371 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.369+0000 7f82b4c0c640 1 -- 172.21.15.105:0/3681912882 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f82b012f650 msgr2=0x7f82b009b4b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-09-21T01:47:00.371 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.369+0000 7f82b4c0c640 1 --2- 172.21.15.105:0/3681912882 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f82b012f650 0x7f82b009b4b0 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7f82a000c430 tx=0x7f82a000c8f0 comp rx=0 tx=0).stop 2024-09-21T01:47:00.371 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.369+0000 7f82b4c0c640 1 -- 172.21.15.105:0/3681912882 shutdown_connections 2024-09-21T01:47:00.371 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.369+0000 7f82b4c0c640 1 --2- 172.21.15.105:0/3681912882 >> [v2:172.21.15.105:3301/0,v1:172.21.15.105:6790/0] conn(0x7f82b0135190 0x7f82b009bf30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:47:00.371 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.369+0000 7f82b4c0c640 1 --2- 172.21.15.105:0/3681912882 >> [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] conn(0x7f82b0130170 0x7f82b009b9f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:47:00.371 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.369+0000 7f82b4c0c640 1 --2- 172.21.15.105:0/3681912882 >> [v2:172.21.15.105:3302/0,v1:172.21.15.105:6791/0] conn(0x7f82b012f650 0x7f82b009b4b0 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-09-21T01:47:00.371 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.369+0000 7f82b4c0c640 1 -- 172.21.15.105:0/3681912882 >> 172.21.15.105:0/3681912882 conn(0x7f82b00a1e40 msgr2=0x7f82b0073ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-09-21T01:47:00.371 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.369+0000 7f82b4c0c640 1 -- 172.21.15.105:0/3681912882 shutdown_connections 2024-09-21T01:47:00.371 INFO:teuthology.orchestra.run.smithi105.stderr:2024-09-21T01:47:00.369+0000 7f82b4c0c640 1 -- 172.21.15.105:0/3681912882 wait complete. 2024-09-21T01:47:00.382 INFO:teuthology.misc:Shutting down mds daemons... 2024-09-21T01:47:00.383 INFO:teuthology.misc:Shutting down osd daemons... 2024-09-21T01:47:00.383 DEBUG:tasks.ceph.osd.0:waiting for process to exit 2024-09-21T01:47:00.383 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T01:47:00.414 INFO:tasks.ceph.osd.0:Stopped 2024-09-21T01:47:00.414 DEBUG:tasks.ceph.osd.1:waiting for process to exit 2024-09-21T01:47:00.414 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T01:47:00.490 INFO:tasks.ceph.osd.1:Stopped 2024-09-21T01:47:00.490 DEBUG:tasks.ceph.osd.2:waiting for process to exit 2024-09-21T01:47:00.490 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T01:47:00.566 INFO:tasks.ceph.osd.2:Stopped 2024-09-21T01:47:00.566 DEBUG:tasks.ceph.osd.3:waiting for process to exit 2024-09-21T01:47:00.566 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T01:47:00.638 INFO:tasks.ceph.osd.3:Stopped 2024-09-21T01:47:00.639 INFO:teuthology.misc:Shutting down mgr daemons... 2024-09-21T01:47:00.639 DEBUG:tasks.ceph.mgr.x:waiting for process to exit 2024-09-21T01:47:00.639 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T01:47:00.700 INFO:tasks.ceph.mgr.x:Stopped 2024-09-21T01:47:00.700 DEBUG:tasks.ceph.mgr.y:waiting for process to exit 2024-09-21T01:47:00.700 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T01:47:00.764 INFO:tasks.ceph.mgr.y:Stopped 2024-09-21T01:47:00.764 INFO:teuthology.misc:Shutting down mon daemons... 2024-09-21T01:47:00.764 DEBUG:tasks.ceph.mon.a:waiting for process to exit 2024-09-21T01:47:00.764 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T01:47:00.816 INFO:tasks.ceph.mon.a:Stopped 2024-09-21T01:47:00.816 DEBUG:tasks.ceph.mon.b:waiting for process to exit 2024-09-21T01:47:00.816 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T01:47:00.868 INFO:tasks.ceph.mon.b:Stopped 2024-09-21T01:47:00.868 DEBUG:tasks.ceph.mon.c:waiting for process to exit 2024-09-21T01:47:00.868 INFO:teuthology.orchestra.run:waiting for 300 2024-09-21T01:47:00.920 INFO:tasks.ceph.mon.c:Stopped 2024-09-21T01:47:00.920 INFO:tasks.ceph:Checking cluster log for badness... 2024-09-21T01:47:00.921 DEBUG:teuthology.orchestra.run.smithi105:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/ceph.log | egrep -v 'overall HEALTH_' | egrep -v '\(FS_' | egrep -v '\(MDS_' | egrep -v '\(OSD_' | egrep -v '\(MON_DOWN\)' | egrep -v '\(CACHE_POOL_' | egrep -v '\(POOL_' | egrep -v '\(MGR_DOWN\)' | egrep -v '\(PG_' | egrep -v '\(SMALLER_PGP_NUM\)' | egrep -v 'Monitor daemon marked osd' | egrep -v 'Behind on trimming' | egrep -v 'Manager daemon' | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v FS_DEGRADED | egrep -v 'fs.*is degraded' | egrep -v 'filesystem is degraded' | egrep -v FS_INLINE_DATA_DEPRECATED | egrep -v FS_WITH_FAILED_MDS | egrep -v MDS_ALL_DOWN | egrep -v 'filesystem is offline' | egrep -v MDS_DAMAGE | egrep -v MDS_DEGRADED | egrep -v MDS_FAILED | egrep -v MDS_INSUFFICIENT_STANDBY | egrep -v 'insufficient standby MDS daemons available' | egrep -v MDS_UP_LESS_THAN_MAX | egrep -v 'online, but wants' | egrep -v 'filesystem is online with fewer MDS than max_mds' | egrep -v POOL_APP_NOT_ENABLED | egrep -v 'do not have an application enabled' | egrep -v 'overall HEALTH_' | egrep -v 'Replacing daemon' | egrep -v 'deprecated feature inline_data' | egrep -v 'overall HEALTH_' | egrep -v '\(OSD_DOWN\)' | egrep -v '\(OSD_' | egrep -v 'but it is still running' | egrep -v 'is not responding' | egrep -v PG_AVAILABILITY | egrep -v PG_DEGRADED | egrep -v 'Reduced data availability' | egrep -v 'Degraded data redundancy' | egrep -v 'scrub mismatch' | egrep -v ScrubResult | egrep -v 'wrongly marked' | egrep -v '\(POOL_APP_NOT_ENABLED\)' | egrep -v '\(SLOW_OPS\)' | egrep -v 'overall HEALTH_' | egrep -v '\(MON_MSGR2_NOT_ENABLED\)' | egrep -v 'slow request' | head -n 1 2024-09-21T01:47:00.999 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-0 on ubuntu@smithi105.front.sepia.ceph.com 2024-09-21T01:47:00.999 DEBUG:teuthology.orchestra.run.smithi105:> sync && sudo umount -f /var/lib/ceph/osd/ceph-0 2024-09-21T01:47:01.196 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-1 on ubuntu@smithi105.front.sepia.ceph.com 2024-09-21T01:47:01.196 DEBUG:teuthology.orchestra.run.smithi105:> sync && sudo umount -f /var/lib/ceph/osd/ceph-1 2024-09-21T01:47:01.316 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-2 on ubuntu@smithi105.front.sepia.ceph.com 2024-09-21T01:47:01.316 DEBUG:teuthology.orchestra.run.smithi105:> sync && sudo umount -f /var/lib/ceph/osd/ceph-2 2024-09-21T01:47:01.366 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/ceph-3 on ubuntu@smithi105.front.sepia.ceph.com 2024-09-21T01:47:01.366 DEBUG:teuthology.orchestra.run.smithi105:> sync && sudo umount -f /var/lib/ceph/osd/ceph-3 2024-09-21T01:47:01.416 INFO:tasks.ceph:Archiving mon data... 2024-09-21T01:47:01.417 DEBUG:teuthology.misc:Transferring archived files from smithi105:/var/lib/ceph/mon/ceph-a to /home/teuthworker/archive/pdonnell-2024-09-20_23:48:22-fs-wip-pdonnell-testing-20240920.212106-debug-distro-default-smithi/7913395/data/mon.a.tgz 2024-09-21T01:47:01.417 DEBUG:teuthology.orchestra.run.smithi105:> mktemp 2024-09-21T01:47:01.432 INFO:teuthology.orchestra.run.smithi105.stdout:/tmp/tmp.TlAoOBc1Tx 2024-09-21T01:47:01.432 DEBUG:teuthology.orchestra.run.smithi105:> sudo tar cz -f - -C /var/lib/ceph/mon/ceph-a -- . > /tmp/tmp.TlAoOBc1Tx 2024-09-21T01:47:01.559 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod 0666 /tmp/tmp.TlAoOBc1Tx 2024-09-21T01:47:01.642 DEBUG:teuthology.orchestra.remote:smithi105:/tmp/tmp.TlAoOBc1Tx is 240KB 2024-09-21T01:47:01.736 DEBUG:teuthology.orchestra.run.smithi105:> rm -fr /tmp/tmp.TlAoOBc1Tx 2024-09-21T01:47:01.750 DEBUG:teuthology.misc:Transferring archived files from smithi105:/var/lib/ceph/mon/ceph-b to /home/teuthworker/archive/pdonnell-2024-09-20_23:48:22-fs-wip-pdonnell-testing-20240920.212106-debug-distro-default-smithi/7913395/data/mon.b.tgz 2024-09-21T01:47:01.750 DEBUG:teuthology.orchestra.run.smithi105:> mktemp 2024-09-21T01:47:01.805 INFO:teuthology.orchestra.run.smithi105.stdout:/tmp/tmp.mq4c3qS0Tq 2024-09-21T01:47:01.805 DEBUG:teuthology.orchestra.run.smithi105:> sudo tar cz -f - -C /var/lib/ceph/mon/ceph-b -- . > /tmp/tmp.mq4c3qS0Tq 2024-09-21T01:47:01.922 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod 0666 /tmp/tmp.mq4c3qS0Tq 2024-09-21T01:47:02.004 DEBUG:teuthology.orchestra.remote:smithi105:/tmp/tmp.mq4c3qS0Tq is 286KB 2024-09-21T01:47:02.097 DEBUG:teuthology.orchestra.run.smithi105:> rm -fr /tmp/tmp.mq4c3qS0Tq 2024-09-21T01:47:02.112 DEBUG:teuthology.misc:Transferring archived files from smithi105:/var/lib/ceph/mon/ceph-c to /home/teuthworker/archive/pdonnell-2024-09-20_23:48:22-fs-wip-pdonnell-testing-20240920.212106-debug-distro-default-smithi/7913395/data/mon.c.tgz 2024-09-21T01:47:02.112 DEBUG:teuthology.orchestra.run.smithi105:> mktemp 2024-09-21T01:47:02.166 INFO:teuthology.orchestra.run.smithi105.stdout:/tmp/tmp.WhHexSmuUh 2024-09-21T01:47:02.167 DEBUG:teuthology.orchestra.run.smithi105:> sudo tar cz -f - -C /var/lib/ceph/mon/ceph-c -- . > /tmp/tmp.WhHexSmuUh 2024-09-21T01:47:02.284 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod 0666 /tmp/tmp.WhHexSmuUh 2024-09-21T01:47:02.366 DEBUG:teuthology.orchestra.remote:smithi105:/tmp/tmp.WhHexSmuUh is 286KB 2024-09-21T01:47:02.460 DEBUG:teuthology.orchestra.run.smithi105:> rm -fr /tmp/tmp.WhHexSmuUh 2024-09-21T01:47:02.475 INFO:tasks.ceph:Cleaning ceph cluster... 2024-09-21T01:47:02.475 DEBUG:teuthology.orchestra.run.smithi105:> sudo rm -rf -- /etc/ceph/ceph.conf /etc/ceph/ceph.keyring /home/ubuntu/cephtest/ceph.data /home/ubuntu/cephtest/ceph.monmap /home/ubuntu/cephtest/../*.pid 2024-09-21T01:47:02.594 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2024-09-21T01:47:02.594 INFO:tasks.ceph:Archiving crash dumps... 2024-09-21T01:47:02.595 DEBUG:teuthology.misc:Transferring archived files from smithi105:/var/lib/ceph/crash to /home/teuthworker/archive/pdonnell-2024-09-20_23:48:22-fs-wip-pdonnell-testing-20240920.212106-debug-distro-default-smithi/7913395/remote/smithi105/crash 2024-09-21T01:47:02.596 DEBUG:teuthology.orchestra.run.smithi105:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2024-09-21T01:47:02.657 INFO:tasks.ceph:Compressing logs... 2024-09-21T01:47:02.658 DEBUG:teuthology.orchestra.run.smithi105:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T01:47:02.755 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-osd.0.log 2024-09-21T01:47:02.755 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32465.log 2024-09-21T01:47:02.755 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.43266.log 2024-09-21T01:47:02.755 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.32465.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47827.log 2024-09-21T01:47:02.755 INFO:teuthology.orchestra.run.smithi105.stderr: 78.2% -- replaced with /var/log/ceph/ceph-client.admin.32465.log.gz 2024-09-21T01:47:02.756 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.43266.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42902.log 2024-09-21T01:47:02.756 INFO:teuthology.orchestra.run.smithi105.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.43266.log.gz 2024-09-21T01:47:02.756 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.47827.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47383.log 2024-09-21T01:47:02.756 INFO:teuthology.orchestra.run.smithi105.stderr: 86.4% -- replaced with /var/log/ceph/ceph-client.admin.47827.log.gz 2024-09-21T01:47:02.756 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.42902.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33451.log 2024-09-21T01:47:02.756 INFO:teuthology.orchestra.run.smithi105.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.42902.log.gz 2024-09-21T01:47:02.756 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.47383.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41996.log 2024-09-21T01:47:02.756 INFO:teuthology.orchestra.run.smithi105.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.47383.log.gz 2024-09-21T01:47:02.756 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.33451.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39449.log 2024-09-21T01:47:02.756 INFO:teuthology.orchestra.run.smithi105.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.33451.log.gz 2024-09-21T01:47:02.756 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.41996.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40155.log 2024-09-21T01:47:02.756 INFO:teuthology.orchestra.run.smithi105.stderr: 87.6% -- replaced with /var/log/ceph/ceph-client.admin.41996.log.gz 2024-09-21T01:47:02.756 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.39449.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47046.log 2024-09-21T01:47:02.756 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.40155.log: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.39449.log.gz 2024-09-21T01:47:02.757 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42533.log 2024-09-21T01:47:02.757 INFO:teuthology.orchestra.run.smithi105.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.40155.log.gz 2024-09-21T01:47:02.757 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.47046.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32948.log 2024-09-21T01:47:02.757 INFO:teuthology.orchestra.run.smithi105.stderr: 85.9% -- replaced with /var/log/ceph/ceph-client.admin.47046.log.gz 2024-09-21T01:47:02.757 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.42533.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32195.log 2024-09-21T01:47:02.757 INFO:teuthology.orchestra.run.smithi105.stderr: 87.1%/var/log/ceph/ceph-client.admin.32948.log: -- replaced with /var/log/ceph/ceph-client.admin.42533.log.gz 2024-09-21T01:47:02.757 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47995.log 2024-09-21T01:47:02.757 INFO:teuthology.orchestra.run.smithi105.stderr: 86.2% -- replaced with /var/log/ceph/ceph-client.admin.32948.log.gz 2024-09-21T01:47:02.757 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.32195.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47659.log 2024-09-21T01:47:02.757 INFO:teuthology.orchestra.run.smithi105.stderr: 0.0% -- replaced with /var/log/ceph/ceph-client.admin.32195.log.gz 2024-09-21T01:47:02.758 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.47995.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.43182.log 2024-09-21T01:47:02.758 INFO:teuthology.orchestra.run.smithi105.stderr: 86.4% -- replaced with /var/log/ceph/ceph-client.admin.47995.log.gz 2024-09-21T01:47:02.758 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39809.log 2024-09-21T01:47:02.758 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.47659.log: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.47659.log.gz 2024-09-21T01:47:02.758 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.43182.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41930.log 2024-09-21T01:47:02.758 INFO:teuthology.orchestra.run.smithi105.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.43182.log.gz 2024-09-21T01:47:02.758 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.39809.log: gzip -5 --verbose -- /var/log/ceph/ceph-osd.1.log 2024-09-21T01:47:02.758 INFO:teuthology.orchestra.run.smithi105.stderr: 85.7% -- replaced with /var/log/ceph/ceph-client.admin.39809.log.gz 2024-09-21T01:47:02.758 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.41930.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42593.log 2024-09-21T01:47:02.758 INFO:teuthology.orchestra.run.smithi105.stderr: 87.3% -- replaced with /var/log/ceph/ceph-client.admin.41930.log.gz 2024-09-21T01:47:02.758 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33371.log 2024-09-21T01:47:02.758 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.42593.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42097.log 2024-09-21T01:47:02.758 INFO:teuthology.orchestra.run.smithi105.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.42593.log.gz 2024-09-21T01:47:02.759 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.33371.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47953.log 2024-09-21T01:47:02.759 INFO:teuthology.orchestra.run.smithi105.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.33371.log.gz 2024-09-21T01:47:02.759 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.42097.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46572.log 2024-09-21T01:47:02.759 INFO:teuthology.orchestra.run.smithi105.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.42097.log.gz 2024-09-21T01:47:02.759 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.47953.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33259.log 2024-09-21T01:47:02.759 INFO:teuthology.orchestra.run.smithi105.stderr: 85.9% -- replaced with /var/log/ceph/ceph-client.admin.47953.log.gz 2024-09-21T01:47:02.759 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42405.log 2024-09-21T01:47:02.759 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.46572.log: /var/log/ceph/ceph-client.admin.33259.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46796.log 2024-09-21T01:47:02.759 INFO:teuthology.orchestra.run.smithi105.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.46572.log.gz 2024-09-21T01:47:02.759 INFO:teuthology.orchestra.run.smithi105.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.33259.log.gz 2024-09-21T01:47:02.759 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.42405.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40105.log 2024-09-21T01:47:02.760 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.46796.log: 87.3% -- replaced with /var/log/ceph/ceph-client.admin.42405.log.gz 2024-09-21T01:47:02.760 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32642.log 2024-09-21T01:47:02.760 INFO:teuthology.orchestra.run.smithi105.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.46796.log.gz 2024-09-21T01:47:02.760 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.40105.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42860.log 2024-09-21T01:47:02.760 INFO:teuthology.orchestra.run.smithi105.stderr: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.40105.log.gz 2024-09-21T01:47:02.760 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.32642.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46732.log 2024-09-21T01:47:02.760 INFO:teuthology.orchestra.run.smithi105.stderr: 86.0% -- replaced with /var/log/ceph/ceph-client.admin.32642.log.gz 2024-09-21T01:47:02.760 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.42860.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42513.log 2024-09-21T01:47:02.760 INFO:teuthology.orchestra.run.smithi105.stderr: 86.3% -- replaced with /var/log/ceph/ceph-client.admin.42860.log.gz 2024-09-21T01:47:02.760 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.46732.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39665.log 2024-09-21T01:47:02.760 INFO:teuthology.orchestra.run.smithi105.stderr: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.46732.log.gz 2024-09-21T01:47:02.760 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.42513.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41912.log 2024-09-21T01:47:02.760 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.39665.log: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.42513.log.gz 2024-09-21T01:47:02.760 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.26333.log 2024-09-21T01:47:02.761 INFO:teuthology.orchestra.run.smithi105.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.39665.log.gz 2024-09-21T01:47:02.761 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.41912.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32901.log 2024-09-21T01:47:02.761 INFO:teuthology.orchestra.run.smithi105.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.41912.log.gz 2024-09-21T01:47:02.761 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph.tmp-client.admin.26333.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.26333.log.gz 2024-09-21T01:47:02.761 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39937.log 2024-09-21T01:47:02.761 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.32901.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47533.log 2024-09-21T01:47:02.761 INFO:teuthology.orchestra.run.smithi105.stderr: 86.2% -- replaced with /var/log/ceph/ceph-client.admin.32901.log.gz 2024-09-21T01:47:02.761 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.39937.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47575.log 2024-09-21T01:47:02.761 INFO:teuthology.orchestra.run.smithi105.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.39937.log.gz 2024-09-21T01:47:02.761 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47362.log 2024-09-21T01:47:02.762 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.47533.log: 86.3% -- replaced with /var/log/ceph/ceph-client.admin.47533.log.gz 2024-09-21T01:47:02.762 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.47575.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32493.log 2024-09-21T01:47:02.762 INFO:teuthology.orchestra.run.smithi105.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.47575.log.gz 2024-09-21T01:47:02.762 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39707.log 2024-09-21T01:47:02.762 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.47362.log: /var/log/ceph/ceph-client.admin.32493.log: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.47362.log.gz 2024-09-21T01:47:02.762 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32946.log 2024-09-21T01:47:02.762 INFO:teuthology.orchestra.run.smithi105.stderr: 78.6% -- replaced with /var/log/ceph/ceph-client.admin.32493.log.gz 2024-09-21T01:47:02.762 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47215.log 2024-09-21T01:47:02.762 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.39707.log: /var/log/ceph/ceph-client.admin.32946.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32857.log 2024-09-21T01:47:02.762 INFO:teuthology.orchestra.run.smithi105.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.39707.log.gz 2024-09-21T01:47:02.762 INFO:teuthology.orchestra.run.smithi105.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.32946.log.gz 2024-09-21T01:47:02.762 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.47215.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39581.log 2024-09-21T01:47:02.763 INFO:teuthology.orchestra.run.smithi105.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.47215.log.gz 2024-09-21T01:47:02.763 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.32857.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.43017.log 2024-09-21T01:47:02.763 INFO:teuthology.orchestra.run.smithi105.stderr: 86.1% -- replaced with /var/log/ceph/ceph-client.admin.32857.log.gz 2024-09-21T01:47:02.763 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.39581.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32943.log 2024-09-21T01:47:02.763 INFO:teuthology.orchestra.run.smithi105.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.39581.log.gz 2024-09-21T01:47:02.763 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.43017.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46903.log 2024-09-21T01:47:02.763 INFO:teuthology.orchestra.run.smithi105.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.43017.log.gz 2024-09-21T01:47:02.763 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.32943.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47214.log 2024-09-21T01:47:02.763 INFO:teuthology.orchestra.run.smithi105.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.32943.log.gz 2024-09-21T01:47:02.763 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.46903.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32343.log 2024-09-21T01:47:02.763 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.47214.log: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.46903.log.gz 2024-09-21T01:47:02.763 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.audit.log 2024-09-21T01:47:02.764 INFO:teuthology.orchestra.run.smithi105.stderr: 87.3% -- replaced with /var/log/ceph/ceph-client.admin.47214.log.gz 2024-09-21T01:47:02.764 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.32343.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40293.log 2024-09-21T01:47:02.764 INFO:teuthology.orchestra.run.smithi105.stderr: 74.8% -- replaced with /var/log/ceph/ceph-client.admin.32343.log.gz 2024-09-21T01:47:02.764 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40585.log 2024-09-21T01:47:02.764 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.40293.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47155.log 2024-09-21T01:47:02.764 INFO:teuthology.orchestra.run.smithi105.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.40293.log.gz 2024-09-21T01:47:02.764 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.40585.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40463.log 2024-09-21T01:47:02.764 INFO:teuthology.orchestra.run.smithi105.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.40585.log.gz 2024-09-21T01:47:02.764 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.47155.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47047.log 2024-09-21T01:47:02.764 INFO:teuthology.orchestra.run.smithi105.stderr: 87.5% -- replaced with /var/log/ceph/ceph-client.admin.47155.log.gz 2024-09-21T01:47:02.764 INFO:teuthology.orchestra.run.smithi105.stderr: 95.0% -- replaced with /var/log/ceph/ceph.audit.log.gz 2024-09-21T01:47:02.765 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.40463.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33609.log 2024-09-21T01:47:02.765 INFO:teuthology.orchestra.run.smithi105.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.40463.log.gz 2024-09-21T01:47:02.765 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.47047.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33219.log 2024-09-21T01:47:02.765 INFO:teuthology.orchestra.run.smithi105.stderr: 86.0% -- replaced with /var/log/ceph/ceph-client.admin.47047.log.gz 2024-09-21T01:47:02.765 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.33609.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48203.log 2024-09-21T01:47:02.765 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.33219.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40292.log 2024-09-21T01:47:02.765 INFO:teuthology.orchestra.run.smithi105.stderr: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.33609.log.gz 2024-09-21T01:47:02.765 INFO:teuthology.orchestra.run.smithi105.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.33219.log.gz 2024-09-21T01:47:02.765 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.48203.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40333.log 2024-09-21T01:47:02.765 INFO:teuthology.orchestra.run.smithi105.stderr: 87.3% -- replaced with /var/log/ceph/ceph-client.admin.48203.log.gz 2024-09-21T01:47:02.765 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.40292.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32685.log 2024-09-21T01:47:02.766 INFO:teuthology.orchestra.run.smithi105.stderr: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.40292.log.gz 2024-09-21T01:47:02.766 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42445.log 2024-09-21T01:47:02.766 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.40333.log: /var/log/ceph/ceph-client.admin.32685.log: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.40333.log.gz 2024-09-21T01:47:02.766 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40253.log 2024-09-21T01:47:02.766 INFO:teuthology.orchestra.run.smithi105.stderr: 86.0% -- replaced with /var/log/ceph/ceph-client.admin.32685.log.gz 2024-09-21T01:47:02.766 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48163.log 2024-09-21T01:47:02.766 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.42445.log: /var/log/ceph/ceph-client.admin.40253.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48079.log 2024-09-21T01:47:02.766 INFO:teuthology.orchestra.run.smithi105.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.42445.log.gz 2024-09-21T01:47:02.766 INFO:teuthology.orchestra.run.smithi105.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.40253.log.gz 2024-09-21T01:47:02.766 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.48163.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47701.log 2024-09-21T01:47:02.766 INFO:teuthology.orchestra.run.smithi105.stderr: 86.3% -- replaced with /var/log/ceph/ceph-client.admin.48163.log.gz 2024-09-21T01:47:02.766 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.48079.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33411.log 2024-09-21T01:47:02.767 INFO:teuthology.orchestra.run.smithi105.stderr: 86.2% -- replaced with /var/log/ceph/ceph-client.admin.48079.log.gz 2024-09-21T01:47:02.767 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41633.log 2024-09-21T01:47:02.767 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.47701.log: /var/log/ceph/ceph-client.admin.33411.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46612.log 2024-09-21T01:47:02.767 INFO:teuthology.orchestra.run.smithi105.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.47701.log.gz 2024-09-21T01:47:02.767 INFO:teuthology.orchestra.run.smithi105.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.33411.log.gz 2024-09-21T01:47:02.767 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.41633.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40154.log 2024-09-21T01:47:02.767 INFO:teuthology.orchestra.run.smithi105.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.41633.log.gz 2024-09-21T01:47:02.767 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47323.log 2024-09-21T01:47:02.767 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.46612.log: gzip -5/var/log/ceph/ceph-client.admin.40154.log: --verbose -- /var/log/ceph/ceph-client.admin.39623.log 2024-09-21T01:47:02.768 INFO:teuthology.orchestra.run.smithi105.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.46612.log.gz 2024-09-21T01:47:02.768 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.47323.log: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.40154.log.gz 2024-09-21T01:47:02.768 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-mon.c.log 2024-09-21T01:47:02.768 INFO:teuthology.orchestra.run.smithi105.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.47323.log.gz 2024-09-21T01:47:02.768 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.39623.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47363.log 2024-09-21T01:47:02.768 INFO:teuthology.orchestra.run.smithi105.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.39623.log.gz 2024-09-21T01:47:02.768 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46945.log 2024-09-21T01:47:02.768 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.47363.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32745.log 2024-09-21T01:47:02.768 INFO:teuthology.orchestra.run.smithi105.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.47363.log.gz 2024-09-21T01:47:02.768 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41985.log 2024-09-21T01:47:02.768 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.46945.log: /var/log/ceph/ceph-client.admin.32745.log: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.46945.log.gz 2024-09-21T01:47:02.769 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47617.log 2024-09-21T01:47:02.769 INFO:teuthology.orchestra.run.smithi105.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.32745.log.gz 2024-09-21T01:47:02.769 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.41985.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33491.log 2024-09-21T01:47:02.769 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.47617.log: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.41985.log.gz 2024-09-21T01:47:02.769 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47785.log 2024-09-21T01:47:02.769 INFO:teuthology.orchestra.run.smithi105.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.47617.log.gz 2024-09-21T01:47:02.769 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.33491.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32893.log 2024-09-21T01:47:02.769 INFO:teuthology.orchestra.run.smithi105.stderr: 86.8%gzip -- replaced with /var/log/ceph/ceph-client.admin.33491.log.gz -5 2024-09-21T01:47:02.769 INFO:teuthology.orchestra.run.smithi105.stderr: --verbose -- /var/log/ceph/ceph-mon.a.log 2024-09-21T01:47:02.769 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.47785.log: /var/log/ceph/ceph-client.admin.32893.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42712.log 2024-09-21T01:47:02.769 INFO:teuthology.orchestra.run.smithi105.stderr: 86.0% -- replaced with /var/log/ceph/ceph-client.admin.47785.log.gz 2024-09-21T01:47:02.769 INFO:teuthology.orchestra.run.smithi105.stderr: 86.2% -- replaced with /var/log/ceph/ceph-client.admin.32893.log.gz 2024-09-21T01:47:02.769 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47491.log 2024-09-21T01:47:02.769 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.42712.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46692.log 2024-09-21T01:47:02.770 INFO:teuthology.orchestra.run.smithi105.stderr: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.42712.log.gz 2024-09-21T01:47:02.770 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.47491.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32437.log 2024-09-21T01:47:02.770 INFO:teuthology.orchestra.run.smithi105.stderr: 86.3% -- replaced with /var/log/ceph/ceph-client.admin.47491.log.gz 2024-09-21T01:47:02.770 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.46692.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32596.log 2024-09-21T01:47:02.770 INFO:teuthology.orchestra.run.smithi105.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.46692.log.gz 2024-09-21T01:47:02.770 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.32437.log: gzip 75.8% -- replaced with /var/log/ceph/ceph-client.admin.32437.log.gz 2024-09-21T01:47:02.770 INFO:teuthology.orchestra.run.smithi105.stderr: -5 --verbose -- /var/log/ceph/ceph-client.admin.46819.log 2024-09-21T01:47:02.770 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.32596.log: gzip -5 --verbose -- /var/log/ceph/ceph-osd.2.log 2024-09-21T01:47:02.770 INFO:teuthology.orchestra.run.smithi105.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.32596.log.gz 2024-09-21T01:47:02.770 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.46819.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39977.log 2024-09-21T01:47:02.770 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-osd.2.log: 86.6%gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39806.log 2024-09-21T01:47:02.770 INFO:teuthology.orchestra.run.smithi105.stderr: -- replaced with /var/log/ceph/ceph-client.admin.46819.log.gz 2024-09-21T01:47:02.770 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.39977.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32850.log 2024-09-21T01:47:02.770 INFO:teuthology.orchestra.run.smithi105.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.39977.log.gz 2024-09-21T01:47:02.771 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47045.log 2024-09-21T01:47:02.771 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.39806.log: /var/log/ceph/ceph-client.admin.32850.log: 86.1% -- replaced with /var/log/ceph/ceph-client.admin.39806.log.gz 2024-09-21T01:47:02.771 INFO:teuthology.orchestra.run.smithi105.stderr: 86.3% -- replaced with /var/log/ceph/ceph-client.admin.32850.log.gz 2024-09-21T01:47:02.771 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42818.log 2024-09-21T01:47:02.771 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.47045.log: gzip -5 --verbose -- /var/log/ceph/ceph-mgr.y.log 2024-09-21T01:47:02.771 INFO:teuthology.orchestra.run.smithi105.stderr: 86.2% -- replaced with /var/log/ceph/ceph-client.admin.47045.log.gz/var/log/ceph/ceph-client.admin.42818.log: 2024-09-21T01:47:02.771 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42117.log 2024-09-21T01:47:02.771 INFO:teuthology.orchestra.run.smithi105.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.42818.log.gz 2024-09-21T01:47:02.800 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41776.log 2024-09-21T01:47:02.800 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.42117.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40545.log 2024-09-21T01:47:02.800 INFO:teuthology.orchestra.run.smithi105.stderr: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.42117.log.gz 2024-09-21T01:47:02.800 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.41776.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47911.log 2024-09-21T01:47:02.801 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.40545.log: 85.8% -- replaced with /var/log/ceph/ceph-client.admin.41776.log.gz 2024-09-21T01:47:02.801 INFO:teuthology.orchestra.run.smithi105.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.40545.log.gz 2024-09-21T01:47:02.801 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39505.log 2024-09-21T01:47:02.801 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.33702.log 2024-09-21T01:47:02.801 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.39505.log: /var/log/ceph/ceph-client.admin.47911.log: 86.0% -- replaced with /var/log/ceph/ceph-client.admin.47911.log.gz 2024-09-21T01:47:02.801 INFO:teuthology.orchestra.run.smithi105.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.39505.log.gz 2024-09-21T01:47:02.801 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46773.log 2024-09-21T01:47:02.801 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.33702.log: gzip -5 --verbose -- /var/log/ceph/ceph.log 2024-09-21T01:47:02.801 INFO:teuthology.orchestra.run.smithi105.stderr: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.33702.log.gz 2024-09-21T01:47:02.801 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.46773.log: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.46773.log.gz 2024-09-21T01:47:02.801 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46861.log 2024-09-21T01:47:02.801 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40085.log 2024-09-21T01:47:02.801 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph.log: /var/log/ceph/ceph-client.admin.46861.log: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.46861.log.gz 2024-09-21T01:47:02.801 INFO:teuthology.orchestra.run.smithi105.stderr: 93.8% -- replaced with /var/log/ceph/ceph.log.gz 2024-09-21T01:47:02.802 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42345.log 2024-09-21T01:47:02.802 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41777.log 2024-09-21T01:47:02.802 INFO:teuthology.orchestra.run.smithi105.stderr: 93.6% -- replaced with /var/log/ceph/ceph-osd.0.log.gz/var/log/ceph/ceph-client.admin.40085.log: 2024-09-21T01:47:02.802 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.42345.log: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.40085.log.gz 2024-09-21T01:47:02.802 INFO:teuthology.orchestra.run.smithi105.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.42345.log.gz 2024-09-21T01:47:02.802 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32876.log 2024-09-21T01:47:02.802 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.41777.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.43094.log 2024-09-21T01:47:02.802 INFO:teuthology.orchestra.run.smithi105.stderr: 85.7% -- replaced with /var/log/ceph/ceph-client.admin.41777.log.gz 2024-09-21T01:47:02.802 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.32876.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42692.log 2024-09-21T01:47:02.802 INFO:teuthology.orchestra.run.smithi105.stderr: 86.3% -- replaced with /var/log/ceph/ceph-client.admin.32876.log.gz 2024-09-21T01:47:02.802 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose --/var/log/ceph/ceph-client.admin.43094.log: /var/log/ceph/ceph-client.admin.42158.log 2024-09-21T01:47:02.802 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.42692.log: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.43094.log.gz 2024-09-21T01:47:02.802 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40421.log 2024-09-21T01:47:02.802 INFO:teuthology.orchestra.run.smithi105.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.42692.log.gz 2024-09-21T01:47:02.803 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.42158.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47869.log 2024-09-21T01:47:02.803 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.40421.log: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.42158.log.gz 2024-09-21T01:47:02.803 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32372.log 2024-09-21T01:47:02.803 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-mgr.x.log 2024-09-21T01:47:02.803 INFO:teuthology.orchestra.run.smithi105.stderr: 86.5% -- replaced with /var/log/ceph/ceph-client.admin.40421.log.gz/var/log/ceph/ceph-client.admin.47869.log: 2024-09-21T01:47:02.803 INFO:teuthology.orchestra.run.smithi105.stderr: 85.7%/var/log/ceph/ceph-client.admin.32372.log: -- replaced with /var/log/ceph/ceph-client.admin.47869.log.gz 2024-09-21T01:47:02.803 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.40505.log 2024-09-21T01:47:02.803 INFO:teuthology.orchestra.run.smithi105.stderr: 78.5% -- replaced with /var/log/ceph/ceph-client.admin.32372.log.gz 2024-09-21T01:47:02.803 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41778.log 2024-09-21T01:47:02.803 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.40505.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48037.log 2024-09-21T01:47:02.803 INFO:teuthology.orchestra.run.smithi105.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.40505.log.gz 2024-09-21T01:47:02.803 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.41778.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32878.log 2024-09-21T01:47:02.803 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.48037.log: 85.1% -- replaced with /var/log/ceph/ceph-client.admin.41778.log.gz 2024-09-21T01:47:02.803 INFO:teuthology.orchestra.run.smithi105.stderr: 86.1% -- replaced with /var/log/ceph/ceph-client.admin.48037.log.gz 2024-09-21T01:47:02.803 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.44965.log 2024-09-21T01:47:02.804 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42944.log 2024-09-21T01:47:02.804 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.44965.log: /var/log/ceph/ceph-client.admin.32878.log: 85.9% -- replaced with /var/log/ceph/ceph-client.admin.32878.log.gz 2024-09-21T01:47:02.804 INFO:teuthology.orchestra.run.smithi105.stderr: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.44965.log.gz 2024-09-21T01:47:02.804 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39341.log 2024-09-21T01:47:02.804 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.42944.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42179.log 2024-09-21T01:47:02.804 INFO:teuthology.orchestra.run.smithi105.stderr: 86.6% -- replaced with /var/log/ceph/ceph-client.admin.42944.log.gz 2024-09-21T01:47:02.804 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.39341.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41775.log 2024-09-21T01:47:02.804 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.42179.log: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.39341.log.gz 2024-09-21T01:47:02.804 INFO:teuthology.orchestra.run.smithi105.stderr: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.42179.log.gz 2024-09-21T01:47:02.804 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32409.log 2024-09-21T01:47:02.804 INFO:teuthology.orchestra.run.smithi105.stderr: 91.8%/var/log/ceph/ceph-client.admin.41775.log: -- replaced with /var/log/ceph/ceph-mgr.x.log.gz 2024-09-21T01:47:02.804 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39976.log 2024-09-21T01:47:02.804 INFO:teuthology.orchestra.run.smithi105.stderr: 85.7% -- replaced with /var/log/ceph/ceph-client.admin.41775.log.gz 2024-09-21T01:47:02.804 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose/var/log/ceph/ceph-client.admin.32409.log: -- /var/log/ceph/ceph-mon.b.log 2024-09-21T01:47:02.804 INFO:teuthology.orchestra.run.smithi105.stderr: 76.1% -- replaced with /var/log/ceph/ceph-client.admin.32409.log.gz/var/log/ceph/ceph-client.admin.39976.log: 2024-09-21T01:47:02.805 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose 86.8% -- /var/log/ceph/ceph-client.admin.42594.log 2024-09-21T01:47:02.805 INFO:teuthology.orchestra.run.smithi105.stderr: -- replaced with /var/log/ceph/ceph-client.admin.39976.log.gz 2024-09-21T01:47:02.805 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-mon.b.log: gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41582.log 2024-09-21T01:47:02.805 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39936.log 2024-09-21T01:47:02.805 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.42594.log: /var/log/ceph/ceph-client.admin.41582.log: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.42594.log.gz 2024-09-21T01:47:02.805 INFO:teuthology.orchestra.run.smithi105.stderr: 85.8% -- replaced with /var/log/ceph/ceph-client.admin.41582.log.gz 2024-09-21T01:47:02.805 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.41676.log 2024-09-21T01:47:02.805 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39530.log 2024-09-21T01:47:02.805 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.41676.log: /var/log/ceph/ceph-client.admin.39936.log: 86.7% -- replaced with /var/log/ceph/ceph-client.admin.39936.log.gz 2024-09-21T01:47:02.805 INFO:teuthology.orchestra.run.smithi105.stderr: 87.1% -- replaced with /var/log/ceph/ceph-client.admin.41676.log.gz 2024-09-21T01:47:02.805 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32913.log 2024-09-21T01:47:02.805 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47743.log 2024-09-21T01:47:02.805 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.39530.log: /var/log/ceph/ceph-client.admin.32913.log: 85.4% -- replaced with /var/log/ceph/ceph-client.admin.39530.log.gz 2024-09-21T01:47:02.805 INFO:teuthology.orchestra.run.smithi105.stderr: 86.2% -- replaced with /var/log/ceph/ceph-client.admin.32913.log.gz 2024-09-21T01:47:02.805 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42365.log 2024-09-21T01:47:02.806 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47044.log 2024-09-21T01:47:02.806 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.47743.log: /var/log/ceph/ceph-client.admin.42365.log: 87.0% -- replaced with /var/log/ceph/ceph-client.admin.47743.log.gz 2024-09-21T01:47:02.806 INFO:teuthology.orchestra.run.smithi105.stderr: 87.2% -- replaced with /var/log/ceph/ceph-client.admin.42365.log.gz 2024-09-21T01:47:02.806 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39807.log 2024-09-21T01:47:02.806 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-osd.3.log 2024-09-21T01:47:02.806 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.39807.log: /var/log/ceph/ceph-client.admin.47044.log: 85.7% -- replaced with /var/log/ceph/ceph-client.admin.47044.log.gz 2024-09-21T01:47:02.806 INFO:teuthology.orchestra.run.smithi105.stderr: 85.6% -- replaced with /var/log/ceph/ceph-client.admin.39807.log.gz 2024-09-21T01:47:02.806 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.42752.log 2024-09-21T01:47:02.806 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.48121.log 2024-09-21T01:47:02.806 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-osd.3.log: /var/log/ceph/ceph-client.admin.42752.log: 86.8% -- replaced with /var/log/ceph/ceph-client.admin.42752.log.gz 2024-09-21T01:47:02.806 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.46652.log 2024-09-21T01:47:02.806 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.48121.log: 85.9% -- replaced with /var/log/ceph/ceph-client.admin.48121.log.gz 2024-09-21T01:47:02.806 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.47213.log 2024-09-21T01:47:02.806 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.46652.log: 87.3% -- replaced with /var/log/ceph/ceph-client.admin.46652.log.gz 2024-09-21T01:47:02.807 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.32172.log 2024-09-21T01:47:02.807 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.47213.log: 86.9% -- replaced with /var/log/ceph/ceph-client.admin.47213.log.gz 2024-09-21T01:47:02.807 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/ceph-client.admin.39808.log 2024-09-21T01:47:02.807 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.32172.log: 0.0% -- replaced with /var/log/ceph/ceph-client.admin.32172.log.gz 2024-09-21T01:47:02.807 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/ceph-client.admin.39808.log: 84.9% -- replaced with /var/log/ceph/ceph-client.admin.39808.log.gz 2024-09-21T01:47:02.807 INFO:teuthology.orchestra.run.smithi105.stderr: 90.5% -- replaced with /var/log/ceph/ceph-mgr.y.log.gz 2024-09-21T01:47:02.807 INFO:teuthology.orchestra.run.smithi105.stderr: 93.2% -- replaced with /var/log/ceph/ceph-osd.2.log.gz 2024-09-21T01:47:02.811 INFO:teuthology.orchestra.run.smithi105.stderr: 91.6% -- replaced with /var/log/ceph/ceph-mon.c.log.gz 2024-09-21T01:47:02.816 INFO:teuthology.orchestra.run.smithi105.stderr: 93.8% -- replaced with /var/log/ceph/ceph-osd.1.log.gz 2024-09-21T01:47:02.864 INFO:teuthology.orchestra.run.smithi105.stderr: 93.3% -- replaced with /var/log/ceph/ceph-osd.3.log.gz 2024-09-21T01:47:02.865 INFO:teuthology.orchestra.run.smithi105.stderr: 91.7% -- replaced with /var/log/ceph/ceph-mon.b.log.gz 2024-09-21T01:47:02.875 INFO:teuthology.orchestra.run.smithi105.stderr: 91.0% -- replaced with /var/log/ceph/ceph-mon.a.log.gz 2024-09-21T01:47:02.877 INFO:teuthology.orchestra.run.smithi105.stderr: 2024-09-21T01:47:02.877 INFO:teuthology.orchestra.run.smithi105.stderr:real 0m0.164s 2024-09-21T01:47:02.877 INFO:teuthology.orchestra.run.smithi105.stderr:user 0m0.624s 2024-09-21T01:47:02.877 INFO:teuthology.orchestra.run.smithi105.stderr:sys 0m0.114s 2024-09-21T01:47:02.877 INFO:tasks.ceph:Archiving logs... 2024-09-21T01:47:02.878 DEBUG:teuthology.misc:Transferring archived files from smithi105:/var/log/ceph to /home/teuthworker/archive/pdonnell-2024-09-20_23:48:22-fs-wip-pdonnell-testing-20240920.212106-debug-distro-default-smithi/7913395/remote/smithi105/log 2024-09-21T01:47:02.878 DEBUG:teuthology.orchestra.run.smithi105:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T01:47:03.128 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-21T01:47:03.148 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-21T01:47:03.148 DEBUG:teuthology.orchestra.run.smithi105:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-21T01:47:03.201 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-21T01:47:03.210 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-21T01:47:03.210 DEBUG:teuthology.orchestra.run.smithi105:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T01:47:03.229 INFO:teuthology.orchestra.run.smithi105.stderr:bash: line 1: ntpq: command not found 2024-09-21T01:47:03.232 INFO:teuthology.orchestra.run.smithi105.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T01:47:03.233 INFO:teuthology.orchestra.run.smithi105.stdout:=============================================================================== 2024-09-21T01:47:03.233 INFO:teuthology.orchestra.run.smithi105.stdout:^* hv01.front.sepia.ceph.com 3 6 177 57 +342us[ +331us] +/- 46ms 2024-09-21T01:47:03.233 INFO:teuthology.orchestra.run.smithi105.stdout:^+ hv02.front.sepia.ceph.com 3 6 177 56 -955us[ -955us] +/- 56ms 2024-09-21T01:47:03.233 INFO:teuthology.orchestra.run.smithi105.stdout:^+ hv03.front.sepia.ceph.com 3 6 177 56 +647us[ +647us] +/- 66ms 2024-09-21T01:47:03.233 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T01:47:03.234 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-21T01:47:03.279 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-21T01:47:03.280 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-21T01:47:03.362 DEBUG:teuthology.orchestra.run.smithi105:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T01:47:03.427 DEBUG:teuthology.orchestra.run.smithi105:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T01:47:03.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T01:47:03.454 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-21T01:47:03.465 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-21T01:47:03.532 INFO:teuthology.task.internal:Duration was 483.551328 seconds 2024-09-21T01:47:03.532 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-21T01:47:03.619 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-21T01:47:03.620 DEBUG:teuthology.orchestra.run.smithi105:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T01:47:03.659 INFO:teuthology.orchestra.run.smithi105.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T01:47:03.871 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-21T01:47:03.871 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi105.front.sepia.ceph.com 2024-09-21T01:47:03.872 DEBUG:teuthology.orchestra.run.smithi105:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T01:47:03.931 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-21T01:47:03.931 DEBUG:teuthology.orchestra.run.smithi105:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T01:47:03.998 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-21T01:47:03.998 DEBUG:teuthology.orchestra.run.smithi105:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T01:47:04.140 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-21T01:47:04.152 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-21T01:47:04.152 DEBUG:teuthology.orchestra.run.smithi105:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T01:47:04.177 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-21T01:47:04.194 DEBUG:teuthology.orchestra.run.smithi105:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T01:47:04.240 INFO:teuthology.orchestra.run.smithi105.stdout:kernel.core_pattern = core 2024-09-21T01:47:04.265 DEBUG:teuthology.orchestra.run.smithi105:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T01:47:04.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T01:47:04.323 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-21T01:47:04.395 INFO:teuthology.task.internal:Transferring archived files... 2024-09-21T01:47:04.396 DEBUG:teuthology.misc:Transferring archived files from smithi105:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/pdonnell-2024-09-20_23:48:22-fs-wip-pdonnell-testing-20240920.212106-debug-distro-default-smithi/7913395/remote/smithi105 2024-09-21T01:47:04.396 DEBUG:teuthology.orchestra.run.smithi105:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T01:47:04.464 INFO:teuthology.task.internal:Removing archive directory... 2024-09-21T01:47:04.465 DEBUG:teuthology.orchestra.run.smithi105:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T01:47:04.515 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-21T01:47:04.535 INFO:teuthology.task.internal:Not uploading archives. 2024-09-21T01:47:04.535 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-21T01:47:04.594 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-21T01:47:04.594 DEBUG:teuthology.orchestra.run.smithi105:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T01:47:04.609 INFO:teuthology.orchestra.run.smithi105.stdout: 265336 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 01:47 /home/ubuntu/cephtest 2024-09-21T01:47:04.611 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-21T01:47:04.685 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-21T01:47:04.965 INFO:teuthology.run:Summary data: description: fs/upgrade/nofs/{bluestore-bitmap centos_9.stream conf/{client mds mgr mon osd} no-mds-cluster overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn pg_health} tasks/{0-from/reef 1-upgrade}} duration: 483.55132818222046 flavor: default owner: scheduled_pdonnell@teuthology success: true 2024-09-21T01:47:04.966 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T01:47:05.065 INFO:teuthology.run:pass